From 7ee45eed858ea2c753593cce613e27d96e214a60 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Fri, 26 Apr 2024 21:28:24 +0530 Subject: [PATCH 01/26] feat: emersys initial commit --- src/cdk/v2/destinations/emersys/config.js | 26 ++++ .../v2/destinations/emersys/procWorkflow.yaml | 57 ++++++++ .../v2/destinations/emersys/rtWorkflow.yaml | 39 ++++++ src/cdk/v2/destinations/emersys/utils.js | 125 ++++++++++++++++++ 4 files changed, 247 insertions(+) create mode 100644 src/cdk/v2/destinations/emersys/config.js create mode 100644 src/cdk/v2/destinations/emersys/procWorkflow.yaml create mode 100644 src/cdk/v2/destinations/emersys/rtWorkflow.yaml create mode 100644 src/cdk/v2/destinations/emersys/utils.js diff --git a/src/cdk/v2/destinations/emersys/config.js b/src/cdk/v2/destinations/emersys/config.js new file mode 100644 index 0000000000..344980e7d0 --- /dev/null +++ b/src/cdk/v2/destinations/emersys/config.js @@ -0,0 +1,26 @@ +const { getMappingConfig } = require('../../../../v0/util'); + +// ref : https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/conversions-api?view=li-lms-2024-02&tabs=http#adding-multiple-conversion-events-in-a-batch +const BATCH_ENDPOINT = 'https://api.linkedin.com/rest/conversionEvents'; +const API_HEADER_METHOD = 'BATCH_CREATE'; +const API_VERSION = '202402'; // yyyymm format +const API_PROTOCOL_VERSION = '2.0.0'; + +const CONFIG_CATEGORIES = { + USER_INFO: { + name: 'linkedinUserInfoConfig', + type: 'user', + }, +}; + +const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); + +module.exports = { + MAX_BATCH_SIZE: 5000, + BATCH_ENDPOINT, + API_HEADER_METHOD, + API_VERSION, + API_PROTOCOL_VERSION, + CONFIG_CATEGORIES, + MAPPING_CONFIG, +}; diff --git a/src/cdk/v2/destinations/emersys/procWorkflow.yaml b/src/cdk/v2/destinations/emersys/procWorkflow.yaml new file mode 100644 index 0000000000..322631a515 --- /dev/null +++ b/src/cdk/v2/destinations/emersys/procWorkflow.yaml @@ -0,0 +1,57 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + exportAll: true + - name: removeUndefinedValues + path: ../../../../v0/util + - name: defaultRequestConfig + path: ../../../../v0/util + - path: ./utils + - path: ./config + - path: lodash + name: cloneDeep + +steps: + - name: checkIfProcessed + condition: .message.statusCode + template: | + $.batchMode ? .message.body.JSON : .message + onComplete: return + - name: messageType + template: | + .message.type.toLowerCase() + - name: validateInput + template: | + let messageType = $.outputs.messageType; + $.assert(messageType, "Message type is not present. Aborting message."); + $.assert(messageType in {{$.EventType.([.TRACK, .IDENTIFY, .GROUP])}}, + "message type " + messageType + " is not supported") + $.assertConfig(.destination.Config.emersysUsername, "Emersys user name is not configured. Aborting"); + $.assertConfig(.destination.Config.emersysUserSecret, "Emersys user secret is not configured. Aborting"); + + - name: validateInputForTrack + description: Additional validation for Track events + condition: $.outputs.messageType === {{$.EventType.TRACK}} + template: | + $.assert(.message.event, "event could not be mapped to conversion rule. Aborting.") + - name: preparePayloadForIdentify + description: | + Builds identify payload. + ref: https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/conversions-api?view=li-lms-2024-02&tabs=curl#adding-multiple-conversion-events-in-a-batch + template: | + $.context.payload = $.buildIdentifyPayload(.message, .destination.Config,); + - name: preparePayloadForGroup + description: | + Builds group payload. + ref: https://dev.emarsys.com/docs/core-api-reference/1m0m70hy3tuov-add-contacts-to-a-contact-list + template: | + $.context.payload = $.buildGroupPayload(.message, .destination.Config,); + - name: buildResponse + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = $.context.payload; + response.endpoint = $.deduceEndPoint(.message,.destination.Config); + response.method = "POST"; + response.headers = $.buildHeader(.destination.Config) + response diff --git a/src/cdk/v2/destinations/emersys/rtWorkflow.yaml b/src/cdk/v2/destinations/emersys/rtWorkflow.yaml new file mode 100644 index 0000000000..dda322e45e --- /dev/null +++ b/src/cdk/v2/destinations/emersys/rtWorkflow.yaml @@ -0,0 +1,39 @@ +bindings: + - path: ./utils + - path: ./config + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + bindings: + - name: batchMode + value: true + loopOverInput: true + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "message": .[], + "destination": ^ [idx].destination, + "metadata": ^ [idx].metadata + })[] + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + $.batchResponseBuilder($.outputs.successfulEvents); + + - name: finalPayload + template: | + [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents] diff --git a/src/cdk/v2/destinations/emersys/utils.js b/src/cdk/v2/destinations/emersys/utils.js new file mode 100644 index 0000000000..4741fb0d36 --- /dev/null +++ b/src/cdk/v2/destinations/emersys/utils.js @@ -0,0 +1,125 @@ +import { EVENT_TYPE } from 'rudder-transformer-cdk/build/constants'; + +const lodash = require('lodash'); +const crypto = require('crypto'); +const get = require('get-value'); + +const { + InstrumentationError, + isDefinedAndNotNullAndNotEmpty, + removeUndefinedAndNullAndEmptyValues, + removeUndefinedAndNullValues, +} = require('@rudderstack/integrations-lib'); +const { getValueFromMessage } = require('rudder-transformer-cdk/build/utils'); +const { getIntegrationsObj } = require('../../../../v0/util'); +const { EMAIL_FIELD_ID } = require('./config'); + +function base64Sha(str) { + const hexDigest = crypto.createHash('sha1').update(str).digest('hex'); + return Buffer.from(hexDigest).toString('base64'); +} + +function getWsseHeader(user, secret) { + const nonce = crypto.randomBytes(16).toString('hex'); + const timestamp = new Date().toISOString(); + + const digest = base64Sha(nonce + timestamp + secret); + return `UsernameToken Username="${user}", PasswordDigest="${digest}", Nonce="${nonce}", Created="${timestamp}"`; +} + +const buildHeader = (destConfig) => { + const { emersysUsername, emersysUserSecret } = destConfig; + return { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': getWsseHeader(emersysUsername, emersysUserSecret), + }; +}; + +const buildIdentifyPayload = (message, destination) => { + let identifyPayload; + const { fieldMapping, emersysCustomIdentifier, discardEmptyProperties, defaultContactList } = + destination.Config; + const payload = {}; + if (fieldMapping) { + fieldMapping.forEach((trait) => { + const { rudderProperty, emersysProperty } = trait; + const value = get(message, rudderProperty); + if (value) { + payload[emersysProperty] = value; + } + }); + } + + const emersysIdentifier = emersysCustomIdentifier || EMAIL_FIELD_ID; + const finalPayload = + discardEmptyProperties === true + ? removeUndefinedAndNullAndEmptyValues(payload) // empty property value has a significance in emersys + : removeUndefinedAndNullValues(payload); + const integrationObject = getIntegrationsObj(message, 'emersys'); + + // TODO: add validation for opt in field + + if (isDefinedAndNotNullAndNotEmpty(payload[emersysIdentifier])) { + identifyPayload = { + key_id: integrationObject.customIdentifierId || emersysIdentifier, + contacts: [...finalPayload], + contact_list_id: integrationObject.contactListId || defaultContactList, + }; + } else { + throw new InstrumentationError( + 'Either configured custom contact identifier value or default identifier email value is missing', + ); + } + + return identifyPayload; +}; + +function findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping) { + // Use lodash to find the object where the emersysProperty matches the input + const item = lodash.find(fieldMapping, { emersysProperty }); + // Return the rudderProperty if the object is found, otherwise return null + return item ? item.rudderProperty : null; +} + +const buildGroupPayload = (message, destination) => { + const { emersysCustomIdentifier, defaultContactList, fieldMapping } = destination.Config; + const integrationObject = getIntegrationsObj(message, 'emersys'); + const emersysIdentifier = emersysCustomIdentifier || EMAIL_FIELD_ID; + const configuredPayloadProperty = findRudderPropertyByEmersysProperty( + emersysIdentifier, + fieldMapping, + ); + const payload = { + key_id: integrationObject.customIdentifierId || emersysIdentifier, + external_ids: [getValueFromMessage(message.context.traits, configuredPayloadProperty)], + }; + return { + payload, + contactListId: message.groupId || defaultContactList, + }; +}; + +const deduceEndPoint = (message, destConfig) => { + let endPoint; + let contactListId; + const { type, groupId } = message; + switch (type) { + case EVENT_TYPE.IDENTIFY: + endPoint = 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1'; + break; + case EVENT_TYPE.GROUP: + contactListId = groupId || destConfig.defaultContactList; + endPoint = `https://api.emarsys.net/api/v2/contactlist/${contactListId}/add`; + break; + default: + break; + } + return endPoint; +}; +module.exports = { + buildIdentifyPayload, + buildGroupPayload, + buildHeader, + deduceEndPoint, +}; From 4db2602bf46f728f0d41eb07128a74f5014c9384 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Sun, 28 Apr 2024 01:15:23 +0530 Subject: [PATCH 02/26] feat: emersys initial batching commit --- src/cdk/v2/destinations/emersys/config.js | 2 +- src/cdk/v2/destinations/emersys/utils.js | 157 ++++++++++++++++++++-- 2 files changed, 147 insertions(+), 12 deletions(-) diff --git a/src/cdk/v2/destinations/emersys/config.js b/src/cdk/v2/destinations/emersys/config.js index 344980e7d0..a15044b19d 100644 --- a/src/cdk/v2/destinations/emersys/config.js +++ b/src/cdk/v2/destinations/emersys/config.js @@ -16,7 +16,7 @@ const CONFIG_CATEGORIES = { const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); module.exports = { - MAX_BATCH_SIZE: 5000, + MAX_BATCH_SIZE: 1000, BATCH_ENDPOINT, API_HEADER_METHOD, API_VERSION, diff --git a/src/cdk/v2/destinations/emersys/utils.js b/src/cdk/v2/destinations/emersys/utils.js index 4741fb0d36..20c6e5924c 100644 --- a/src/cdk/v2/destinations/emersys/utils.js +++ b/src/cdk/v2/destinations/emersys/utils.js @@ -9,10 +9,11 @@ const { isDefinedAndNotNullAndNotEmpty, removeUndefinedAndNullAndEmptyValues, removeUndefinedAndNullValues, + isDefinedAndNotNull, } = require('@rudderstack/integrations-lib'); const { getValueFromMessage } = require('rudder-transformer-cdk/build/utils'); const { getIntegrationsObj } = require('../../../../v0/util'); -const { EMAIL_FIELD_ID } = require('./config'); +const { EMAIL_FIELD_ID, MAX_BATCH_SIZE } = require('./config'); function base64Sha(str) { const hexDigest = crypto.createHash('sha1').update(str).digest('hex'); @@ -37,7 +38,7 @@ const buildHeader = (destConfig) => { }; const buildIdentifyPayload = (message, destination) => { - let identifyPayload; + let destinationPayload; const { fieldMapping, emersysCustomIdentifier, discardEmptyProperties, defaultContactList } = destination.Config; const payload = {}; @@ -61,7 +62,7 @@ const buildIdentifyPayload = (message, destination) => { // TODO: add validation for opt in field if (isDefinedAndNotNullAndNotEmpty(payload[emersysIdentifier])) { - identifyPayload = { + destinationPayload = { key_id: integrationObject.customIdentifierId || emersysIdentifier, contacts: [...finalPayload], contact_list_id: integrationObject.contactListId || defaultContactList, @@ -72,7 +73,7 @@ const buildIdentifyPayload = (message, destination) => { ); } - return identifyPayload; + return { eventType: message.type, destinationPayload }; }; function findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping) { @@ -85,22 +86,30 @@ function findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping) { const buildGroupPayload = (message, destination) => { const { emersysCustomIdentifier, defaultContactList, fieldMapping } = destination.Config; const integrationObject = getIntegrationsObj(message, 'emersys'); - const emersysIdentifier = emersysCustomIdentifier || EMAIL_FIELD_ID; + const emersysIdentifier = + integrationObject.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; const configuredPayloadProperty = findRudderPropertyByEmersysProperty( emersysIdentifier, fieldMapping, ); + const externalIdValue = getValueFromMessage(message.context.traits, configuredPayloadProperty); + if (!isDefinedAndNotNull(externalIdValue)) { + throw new InstrumentationError(''); + } const payload = { - key_id: integrationObject.customIdentifierId || emersysIdentifier, - external_ids: [getValueFromMessage(message.context.traits, configuredPayloadProperty)], + key_id: emersysIdentifier, + external_ids: [externalIdValue], }; return { - payload, - contactListId: message.groupId || defaultContactList, + eventType: message.type, + destinationPayload: { + payload, + contactListId: message.groupId || defaultContactList, + }, }; }; -const deduceEndPoint = (message, destConfig) => { +const deduceEndPoint = (message, destConfig, batchGroupId = undefined) => { let endPoint; let contactListId; const { type, groupId } = message; @@ -109,7 +118,7 @@ const deduceEndPoint = (message, destConfig) => { endPoint = 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1'; break; case EVENT_TYPE.GROUP: - contactListId = groupId || destConfig.defaultContactList; + contactListId = batchGroupId || groupId || destConfig.defaultContactList; endPoint = `https://api.emarsys.net/api/v2/contactlist/${contactListId}/add`; break; default: @@ -117,9 +126,135 @@ const deduceEndPoint = (message, destConfig) => { } return endPoint; }; + +function createIdentifyBatches(events) { + // Grouping the payloads based on key_id and contact_list_id + const groupedIdentifyPayload = lodash.groupBy( + events, + (item) => + `${item.message.body.JSON.destinationPayload.key_id}-${item.message.body.JSON.destinationPayload.contact_list_id}`, + ); + + // Combining the contacts within each group and maintaining the payload structure + const combinedPayloads = Object.keys(groupedIdentifyPayload).map((key) => { + const group = groupedIdentifyPayload[key]; + + // Reduce the group to a single payload with combined contacts + const combinedContacts = group.reduce( + (acc, item) => acc.concat(item.message.body.JSON.destinationPayload.contacts), + [], + ); + + // Use the first item to extract key_id and contact_list_id + const firstItem = group[0].message.body.JSON.destinationPayload; + + return { + key_id: firstItem.key_id, + contacts: combinedContacts, + contact_list_id: firstItem.contact_list_id, + }; + }); + + return combinedPayloads; +} + +function createGroupBatches(events) { + const grouped = lodash.groupBy( + events, + (item) => + `${item.message.body.JSON.destinationPayload.payload.key_id}-${item.message.body.JSON.destinationPayload.contactListId}`, + ); + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + return Object.entries(grouped).map(([key, group]) => { + const keyId = group[0].message.body.JSON.destinationPayload.payload.key_id; + const { contactListId } = group[0].message.body.JSON.destinationPayload; + const combinedExternalIds = group.reduce((acc, item) => { + const ids = item.message.body.JSON.destinationPayload.payload.external_ids; + return acc.concat(ids); + }, []); + + return { + endpoint: `https://api.emarsys.net/api/v2/contactlist/${contactListId}/add`, + payload: { + key_id: keyId, + external_ids: combinedExternalIds, + }, + }; + }); +} +function formatPayloadsWithEndpoint(combinedPayloads, endpointUrl = '') { + return combinedPayloads.map((payload) => ({ + endpoint: endpointUrl, // You can dynamically determine or pass this value + payload, + })); +} + +function batchResponseBuilder(successfulEvents) { + const groupedSuccessfulPayload = { + identify: {}, + group: {}, + }; + let batchesOfIdentifyEvents; + if (successfulEvents.length === 0) { + return []; + } + const constants = { + version: successfulEvents[0].message[0].version, + type: successfulEvents[0].message[0].type, + method: successfulEvents[0].message[0].method, + headers: successfulEvents[0].message[0].headers, + destination: successfulEvents[0].destination, + }; + + const typedEventGroups = lodash.groupBy( + successfulEvents, + (event) => event.message.body.JSON.eventType, + ); + Object.keys(typedEventGroups).forEach((eachEventGroup) => { + switch (eachEventGroup) { + case EVENT_TYPE.IDENTIFY: + batchesOfIdentifyEvents = createIdentifyBatches(eachEventGroup); + groupedSuccessfulPayload.identify = formatPayloadsWithEndpoint( + batchesOfIdentifyEvents, + 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + ); + break; + case EVENT_TYPE.GROUP: + groupedSuccessfulPayload.group = createGroupBatches(eachEventGroup); + break; + default: + break; + } + return groupedSuccessfulPayload; + }); + + return chunkedElements.map((elementsBatch, index) => ({ + batchedRequest: { + body: { + JSON: { elements: elementsBatch }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: constants.version, + type: constants.type, + method: constants.method, + endpoint: constants.endpoint, + headers: constants.headers, + params: {}, + files: {}, + }, + metadata: chunkedMetadata[index], + batched: true, + statusCode: 200, + destination: constants.destination, + })); +} module.exports = { buildIdentifyPayload, buildGroupPayload, buildHeader, deduceEndPoint, + batchResponseBuilder, }; From 0a608b85226b15afa70b96f1dbc27142c3dc5043 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Sun, 28 Apr 2024 12:44:39 +0530 Subject: [PATCH 03/26] feat: emersys initial batching commit 2 --- src/cdk/v2/destinations/emersys/utils.js | 162 ++++++++++++++++------- 1 file changed, 112 insertions(+), 50 deletions(-) diff --git a/src/cdk/v2/destinations/emersys/utils.js b/src/cdk/v2/destinations/emersys/utils.js index 20c6e5924c..c943b90ad2 100644 --- a/src/cdk/v2/destinations/emersys/utils.js +++ b/src/cdk/v2/destinations/emersys/utils.js @@ -127,35 +127,66 @@ const deduceEndPoint = (message, destConfig, batchGroupId = undefined) => { return endPoint; }; +function estimateJsonSize(obj) { + return new Blob([JSON.stringify(obj)]).size; +} + +function createPayload(keyId, contacts, contactListId) { + return { key_id: keyId, contacts, contact_list_id: contactListId }; +} + +function ensureSizeConstraints(contacts) { + const MAX_SIZE_BYTES = 8000000; // 8 MB + const chunks = []; + let currentBatch = []; + + contacts.forEach((contact) => { + // Start a new batch if adding the next contact exceeds size limits + if ( + currentBatch.length === 0 || + estimateJsonSize([...currentBatch, contact]) < MAX_SIZE_BYTES + ) { + currentBatch.push(contact); + } else { + chunks.push(currentBatch); + currentBatch = [contact]; + } + }); + + // Add the remaining batch if not empty + if (currentBatch.length > 0) { + chunks.push(currentBatch); + } + + return chunks; +} + function createIdentifyBatches(events) { - // Grouping the payloads based on key_id and contact_list_id + // Grouping payloads by key_id and contact_list_id const groupedIdentifyPayload = lodash.groupBy( events, (item) => `${item.message.body.JSON.destinationPayload.key_id}-${item.message.body.JSON.destinationPayload.contact_list_id}`, ); - // Combining the contacts within each group and maintaining the payload structure - const combinedPayloads = Object.keys(groupedIdentifyPayload).map((key) => { - const group = groupedIdentifyPayload[key]; + // Process each group + return lodash.flatMap(groupedIdentifyPayload, (group) => { + const firstItem = group[0].message.body.JSON.destinationPayload; + // eslint-disable-next-line @typescript-eslint/naming-convention + const { key_id, contact_list_id } = firstItem; - // Reduce the group to a single payload with combined contacts - const combinedContacts = group.reduce( - (acc, item) => acc.concat(item.message.body.JSON.destinationPayload.contacts), - [], + // Collect all contacts + const allContacts = lodash.flatMap( + group, + (item) => item.message.body.JSON.destinationPayload.contacts, ); + // Chunk by the number of items first, then size + const initialChunks = lodash.chunk(allContacts, MAX_BATCH_SIZE); + const finalChunks = lodash.flatMap(initialChunks, ensureSizeConstraints); - // Use the first item to extract key_id and contact_list_id - const firstItem = group[0].message.body.JSON.destinationPayload; - - return { - key_id: firstItem.key_id, - contacts: combinedContacts, - contact_list_id: firstItem.contact_list_id, - }; + // Create payloads for each chunk + return finalChunks.map((contacts) => createPayload(key_id, contacts, contact_list_id)); }); - - return combinedPayloads; } function createGroupBatches(events) { @@ -166,7 +197,7 @@ function createGroupBatches(events) { ); // eslint-disable-next-line @typescript-eslint/no-unused-vars - return Object.entries(grouped).map(([key, group]) => { + return Object.entries(grouped).flatMap(([key, group]) => { const keyId = group[0].message.body.JSON.destinationPayload.payload.key_id; const { contactListId } = group[0].message.body.JSON.destinationPayload; const combinedExternalIds = group.reduce((acc, item) => { @@ -174,26 +205,59 @@ function createGroupBatches(events) { return acc.concat(ids); }, []); - return { + const idChunks = lodash.chunk(combinedExternalIds, MAX_BATCH_SIZE); + // Map each chunk to a payload configuration + return idChunks.map((chunk) => ({ endpoint: `https://api.emarsys.net/api/v2/contactlist/${contactListId}/add`, payload: { key_id: keyId, - external_ids: combinedExternalIds, + external_ids: chunk, }, - }; + })); }); } -function formatPayloadsWithEndpoint(combinedPayloads, endpointUrl = '') { +function formatIdentifyPayloadsWithEndpoint(combinedPayloads, endpointUrl = '') { return combinedPayloads.map((payload) => ({ - endpoint: endpointUrl, // You can dynamically determine or pass this value + endpoint: endpointUrl, payload, })); } +function buildBatchedRequest(batches, method, constants) { + return batches.map((batch) => ({ + batchedRequest: { + body: { + JSON: batch.payload, // Directly use the payload from the batch + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: constants.version, + type: constants.type, + method, + endpoint: batch.endpoint, + headers: constants.headers, + params: {}, + files: {}, + }, + metadata: chunkedMetadata, + batched: true, + statusCode: 200, + destination: constants.destination, + })); +} + function batchResponseBuilder(successfulEvents) { + const finaloutput = []; const groupedSuccessfulPayload = { - identify: {}, - group: {}, + identify: { + method: 'PUT', + batches: [], + }, + group: { + method: 'POST', + batches: [], + }, }; let batchesOfIdentifyEvents; if (successfulEvents.length === 0) { @@ -202,7 +266,6 @@ function batchResponseBuilder(successfulEvents) { const constants = { version: successfulEvents[0].message[0].version, type: successfulEvents[0].message[0].type, - method: successfulEvents[0].message[0].method, headers: successfulEvents[0].message[0].headers, destination: successfulEvents[0].destination, }; @@ -215,41 +278,40 @@ function batchResponseBuilder(successfulEvents) { switch (eachEventGroup) { case EVENT_TYPE.IDENTIFY: batchesOfIdentifyEvents = createIdentifyBatches(eachEventGroup); - groupedSuccessfulPayload.identify = formatPayloadsWithEndpoint( + groupedSuccessfulPayload.identify.batches = formatIdentifyPayloadsWithEndpoint( batchesOfIdentifyEvents, 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', ); break; case EVENT_TYPE.GROUP: - groupedSuccessfulPayload.group = createGroupBatches(eachEventGroup); + groupedSuccessfulPayload.group.batches = createGroupBatches(eachEventGroup); break; default: break; } return groupedSuccessfulPayload; }); + // Process each identify batch + if (groupedSuccessfulPayload.identify) { + const identifyBatches = buildBatchedRequest( + groupedSuccessfulPayload.identify.batches, + groupedSuccessfulPayload.identify.method, + constants, + ); + finaloutput.push(...identifyBatches); + } - return chunkedElements.map((elementsBatch, index) => ({ - batchedRequest: { - body: { - JSON: { elements: elementsBatch }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, - }, - version: constants.version, - type: constants.type, - method: constants.method, - endpoint: constants.endpoint, - headers: constants.headers, - params: {}, - files: {}, - }, - metadata: chunkedMetadata[index], - batched: true, - statusCode: 200, - destination: constants.destination, - })); + // Process each group batch + if (groupedSuccessfulPayload.group) { + const groupBatches = buildBatchedRequest( + groupedSuccessfulPayload.group.batches, + groupedSuccessfulPayload.group.method, + constants, + ); + finaloutput.push(...groupBatches); + } + + return finaloutput; } module.exports = { buildIdentifyPayload, From 33b87be9b0efdf995af366aae200ad3b8ed4ed6d Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 29 Apr 2024 17:39:20 +0530 Subject: [PATCH 04/26] feat: unit test cases with minor edits --- src/cdk/v2/destinations/emersys/config.js | 26 +- .../v2/destinations/emersys/procWorkflow.yaml | 2 +- src/cdk/v2/destinations/emersys/utils.js | 74 +++-- src/cdk/v2/destinations/emersys/utils.test.js | 290 ++++++++++++++++++ src/constants/destinationCanonicalNames.js | 1 + 5 files changed, 346 insertions(+), 47 deletions(-) create mode 100644 src/cdk/v2/destinations/emersys/utils.test.js diff --git a/src/cdk/v2/destinations/emersys/config.js b/src/cdk/v2/destinations/emersys/config.js index a15044b19d..604cac8cec 100644 --- a/src/cdk/v2/destinations/emersys/config.js +++ b/src/cdk/v2/destinations/emersys/config.js @@ -1,26 +1,8 @@ -const { getMappingConfig } = require('../../../../v0/util'); - -// ref : https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/conversions-api?view=li-lms-2024-02&tabs=http#adding-multiple-conversion-events-in-a-batch -const BATCH_ENDPOINT = 'https://api.linkedin.com/rest/conversionEvents'; -const API_HEADER_METHOD = 'BATCH_CREATE'; -const API_VERSION = '202402'; // yyyymm format -const API_PROTOCOL_VERSION = '2.0.0'; - -const CONFIG_CATEGORIES = { - USER_INFO: { - name: 'linkedinUserInfoConfig', - type: 'user', - }, -}; - -const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); +const ALLOWED_OPT_IN_VALUES = ['1', '2', '']; module.exports = { MAX_BATCH_SIZE: 1000, - BATCH_ENDPOINT, - API_HEADER_METHOD, - API_VERSION, - API_PROTOCOL_VERSION, - CONFIG_CATEGORIES, - MAPPING_CONFIG, + EMAIL_FIELD_ID: 3, + OPT_IN_FILED_ID: 31, + ALLOWED_OPT_IN_VALUES, }; diff --git a/src/cdk/v2/destinations/emersys/procWorkflow.yaml b/src/cdk/v2/destinations/emersys/procWorkflow.yaml index 322631a515..7b4ba51529 100644 --- a/src/cdk/v2/destinations/emersys/procWorkflow.yaml +++ b/src/cdk/v2/destinations/emersys/procWorkflow.yaml @@ -38,7 +38,7 @@ steps: - name: preparePayloadForIdentify description: | Builds identify payload. - ref: https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/conversions-api?view=li-lms-2024-02&tabs=curl#adding-multiple-conversion-events-in-a-batch + ref: template: | $.context.payload = $.buildIdentifyPayload(.message, .destination.Config,); - name: preparePayloadForGroup diff --git a/src/cdk/v2/destinations/emersys/utils.js b/src/cdk/v2/destinations/emersys/utils.js index c943b90ad2..f1aa5cdf2b 100644 --- a/src/cdk/v2/destinations/emersys/utils.js +++ b/src/cdk/v2/destinations/emersys/utils.js @@ -2,7 +2,6 @@ import { EVENT_TYPE } from 'rudder-transformer-cdk/build/constants'; const lodash = require('lodash'); const crypto = require('crypto'); -const get = require('get-value'); const { InstrumentationError, @@ -13,7 +12,12 @@ const { } = require('@rudderstack/integrations-lib'); const { getValueFromMessage } = require('rudder-transformer-cdk/build/utils'); const { getIntegrationsObj } = require('../../../../v0/util'); -const { EMAIL_FIELD_ID, MAX_BATCH_SIZE } = require('./config'); +const { + EMAIL_FIELD_ID, + MAX_BATCH_SIZE, + OPT_IN_FILED_ID, + ALLOWED_OPT_IN_VALUES, +} = require('./config'); function base64Sha(str) { const hexDigest = crypto.createHash('sha1').update(str).digest('hex'); @@ -42,30 +46,46 @@ const buildIdentifyPayload = (message, destination) => { const { fieldMapping, emersysCustomIdentifier, discardEmptyProperties, defaultContactList } = destination.Config; const payload = {}; + + const integrationObject = getIntegrationsObj(message, 'emersys'); + const finalContactList = integrationObject?.contactListId || defaultContactList; + + if (!isDefinedAndNotNullAndNotEmpty(finalContactList)) { + throw new InstrumentationError( + 'Cannot a find a specific contact list either through configuration or via integrations object', + ); + } if (fieldMapping) { fieldMapping.forEach((trait) => { const { rudderProperty, emersysProperty } = trait; - const value = get(message, rudderProperty); + const value = + getValueFromMessage(message.traits, rudderProperty) || + getValueFromMessage(message.context.traits, rudderProperty); if (value) { payload[emersysProperty] = value; } }); } - - const emersysIdentifier = emersysCustomIdentifier || EMAIL_FIELD_ID; + const emersysIdentifier = + integrationObject?.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; const finalPayload = discardEmptyProperties === true ? removeUndefinedAndNullAndEmptyValues(payload) // empty property value has a significance in emersys : removeUndefinedAndNullValues(payload); - const integrationObject = getIntegrationsObj(message, 'emersys'); - - // TODO: add validation for opt in field + if ( + isDefinedAndNotNull(finalPayload[OPT_IN_FILED_ID]) && + !ALLOWED_OPT_IN_VALUES.includes(String(finalPayload[OPT_IN_FILED_ID])) + ) { + throw new InstrumentationError( + `Only ${ALLOWED_OPT_IN_VALUES} values are allowed for optin field`, + ); + } if (isDefinedAndNotNullAndNotEmpty(payload[emersysIdentifier])) { destinationPayload = { - key_id: integrationObject.customIdentifierId || emersysIdentifier, - contacts: [...finalPayload], - contact_list_id: integrationObject.contactListId || defaultContactList, + key_id: emersysIdentifier, + contacts: [finalPayload], + contact_list_id: finalContactList, }; } else { throw new InstrumentationError( @@ -87,14 +107,16 @@ const buildGroupPayload = (message, destination) => { const { emersysCustomIdentifier, defaultContactList, fieldMapping } = destination.Config; const integrationObject = getIntegrationsObj(message, 'emersys'); const emersysIdentifier = - integrationObject.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; + integrationObject?.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; const configuredPayloadProperty = findRudderPropertyByEmersysProperty( emersysIdentifier, fieldMapping, ); const externalIdValue = getValueFromMessage(message.context.traits, configuredPayloadProperty); if (!isDefinedAndNotNull(externalIdValue)) { - throw new InstrumentationError(''); + throw new InstrumentationError( + `No value found in payload for contact custom identifier of id ${emersysIdentifier}`, + ); } const payload = { key_id: emersysIdentifier, @@ -162,30 +184,29 @@ function ensureSizeConstraints(contacts) { } function createIdentifyBatches(events) { - // Grouping payloads by key_id and contact_list_id const groupedIdentifyPayload = lodash.groupBy( events, (item) => `${item.message.body.JSON.destinationPayload.key_id}-${item.message.body.JSON.destinationPayload.contact_list_id}`, ); - // Process each group return lodash.flatMap(groupedIdentifyPayload, (group) => { const firstItem = group[0].message.body.JSON.destinationPayload; // eslint-disable-next-line @typescript-eslint/naming-convention const { key_id, contact_list_id } = firstItem; - // Collect all contacts const allContacts = lodash.flatMap( group, (item) => item.message.body.JSON.destinationPayload.contacts, ); - // Chunk by the number of items first, then size const initialChunks = lodash.chunk(allContacts, MAX_BATCH_SIZE); const finalChunks = lodash.flatMap(initialChunks, ensureSizeConstraints); - // Create payloads for each chunk - return finalChunks.map((contacts) => createPayload(key_id, contacts, contact_list_id)); + // Include metadata for each chunk + return finalChunks.map((contacts) => ({ + payload: createPayload(key_id, contacts, contact_list_id), + metadata: group.map((g) => g.metadata), // assuming metadata is needed per original event grouping + })); }); } @@ -201,18 +222,19 @@ function createGroupBatches(events) { const keyId = group[0].message.body.JSON.destinationPayload.payload.key_id; const { contactListId } = group[0].message.body.JSON.destinationPayload; const combinedExternalIds = group.reduce((acc, item) => { - const ids = item.message.body.JSON.destinationPayload.payload.external_ids; - return acc.concat(ids); + acc.push(...item.message.body.JSON.destinationPayload.payload.external_ids); + return acc; }, []); const idChunks = lodash.chunk(combinedExternalIds, MAX_BATCH_SIZE); - // Map each chunk to a payload configuration + return idChunks.map((chunk) => ({ endpoint: `https://api.emarsys.net/api/v2/contactlist/${contactListId}/add`, payload: { key_id: keyId, external_ids: chunk, }, + metadata: group.map((g) => g.metadata), // assuming metadata is needed per original event grouping })); }); } @@ -227,7 +249,7 @@ function buildBatchedRequest(batches, method, constants) { return batches.map((batch) => ({ batchedRequest: { body: { - JSON: batch.payload, // Directly use the payload from the batch + JSON: batch.payload, JSON_ARRAY: {}, XML: {}, FORM: {}, @@ -240,7 +262,7 @@ function buildBatchedRequest(batches, method, constants) { params: {}, files: {}, }, - metadata: chunkedMetadata, + metadata: batch.metadata, batched: true, statusCode: 200, destination: constants.destination, @@ -319,4 +341,8 @@ module.exports = { buildHeader, deduceEndPoint, batchResponseBuilder, + base64Sha, + getWsseHeader, + findRudderPropertyByEmersysProperty, + formatIdentifyPayloadsWithEndpoint, }; diff --git a/src/cdk/v2/destinations/emersys/utils.test.js b/src/cdk/v2/destinations/emersys/utils.test.js new file mode 100644 index 0000000000..d45171a82c --- /dev/null +++ b/src/cdk/v2/destinations/emersys/utils.test.js @@ -0,0 +1,290 @@ +const { EVENT_TYPE } = require('rudder-transformer-cdk/build/constants'); +const { + buildIdentifyPayload, + buildGroupPayload, + deduceEndPoint, + batchResponseBuilder, + base64Sha, + getWsseHeader, + findRudderPropertyByEmersysProperty, + formatIdentifyPayloadsWithEndpoint, +} = require('./utils'); +const crypto = require('crypto'); +const { InstrumentationError } = require('@rudderstack/integrations-lib'); + +describe('base64Sha', () => { + it('should return a base64 encoded SHA1 hash of the input string', () => { + const input = 'test'; + const expected = 'YTk0YThmZTVjY2IxOWJhNjFjNGMwODczZDM5MWU5ODc5ODJmYmJkMw=='; + const result = base64Sha(input); + expect(result).toEqual(expected); + }); + + it('should return an empty string when input is empty', () => { + const input = ''; + const expected = 'ZGEzOWEzZWU1ZTZiNGIwZDMyNTViZmVmOTU2MDE4OTBhZmQ4MDcwOQ=='; + const result = base64Sha(input); + expect(result).toEqual(expected); + }); +}); + +describe('getWsseHeader', () => { + beforeEach(() => { + jest + .spyOn(crypto, 'randomBytes') + .mockReturnValue(Buffer.from('abcdef1234567890abcdef1234567890', 'hex')); + jest.spyOn(Date.prototype, 'toISOString').mockReturnValue('2024-04-28T12:34:56.789Z'); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should generate a correct WSSE header', () => { + const user = 'testUser'; + const secret = 'testSecret'; + const expectedNonce = 'abcdef1234567890abcdef1234567890'; + const expectedTimestamp = '2024-04-28T12:34:56.789Z'; + const expectedDigest = base64Sha(expectedNonce + expectedTimestamp + secret); + const expectedHeader = `UsernameToken Username="${user}", PasswordDigest="${expectedDigest}", Nonce="${expectedNonce}", Created="${expectedTimestamp}"`; + const result = getWsseHeader(user, secret); + + expect(result).toBe(expectedHeader); + }); +}); + +describe('buildIdentifyPayload', () => { + it('should correctly build payload with field mapping', () => { + const message = { + type: 'identify', + traits: { + firstName: 'John', + lastName: 'Doe', + email: 'john.doe@example.com', + optin: 1, + }, + }; + const destination = { + Config: { + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + defaultContactList: 'dummyContactList', + }, + }; + const expectedPayload = { + contact_list_id: 'dummyContactList', + contacts: [ + { + 1: 'John', + 2: 'Doe', + 3: 'john.doe@example.com', + 31: 1, + }, + ], + key_id: 3, + }; + + const result = buildIdentifyPayload(message, destination); + + expect(result.eventType).toBe(EVENT_TYPE.IDENTIFY); + expect(result.destinationPayload).toEqual(expectedPayload); + }); + + it('should throw error when opt-in field value is not allowed', () => { + const message = { + type: 'identify', + traits: { + firstName: 'John', + lastName: 'Doe', + email: 'john.doe@example.com', + optin: 3, + }, + }; + const destination = { + Config: { + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + defaultContactList: 'dummyList', + }, + }; + expect(() => { + buildIdentifyPayload(message, destination); + }).toThrow('Only 1,2, values are allowed for optin field'); + }); + + it('should throw error when no contact list can be assigned field value is not allowed', () => { + const message = { + type: 'identify', + traits: { + firstName: 'John', + lastName: 'Doe', + email: 'john.doe@example.com', + optin: 1, + }, + }; + const destination = { + Config: { + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + }, + }; + expect(() => { + buildIdentifyPayload(message, destination); + }).toThrow( + 'Cannot a find a specific contact list either through configuration or via integrations object', + ); + }); + + it('should correctly build payload with field mapping present in integrations object', () => { + const message = { + type: 'identify', + traits: { + firstName: 'John', + lastName: 'Doe', + email: 'john.doe@example.com', + optin: 1, + }, + integrations: { + EMERSYS: { + customIdentifierId: 1, + contactListId: 'objectListId', + }, + }, + }; + const destination = { + Config: { + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + defaultContactList: 'dummyContactList', + }, + }; + const expectedPayload = { + contact_list_id: 'objectListId', + contacts: [ + { + 1: 'John', + 2: 'Doe', + 3: 'john.doe@example.com', + 31: 1, + }, + ], + key_id: 1, + }; + + const result = buildIdentifyPayload(message, destination); + + expect(result.eventType).toBe(EVENT_TYPE.IDENTIFY); + expect(result.destinationPayload).toEqual(expectedPayload); + }); +}); + +describe('buildGroupPayload', () => { + // Returns an object with eventType and destinationPayload keys when given valid message and destination inputs + it('should return an object with eventType and destinationPayload keys when given valid message and destination inputs with default externalId', () => { + const message = { + type: 'group', + groupId: 'group123', + context: { + traits: { + email: 'test@example.com', + }, + }, + }; + const destination = { + Config: { + emersysCustomIdentifier: '3', + defaultContactList: 'list123', + fieldMapping: [ + { emersysProperty: '100', rudderProperty: 'customId' }, + { emersysProperty: '3', rudderProperty: 'email' }, + ], + }, + }; + const result = buildGroupPayload(message, destination); + expect(result).toEqual({ + eventType: 'group', + destinationPayload: { + payload: { + key_id: '3', + external_ids: ['test@example.com'], + }, + contactListId: 'group123', + }, + }); + }); + + it('should return an object with eventType and destinationPayload keys when given valid message and destination inputs with configured externalId', () => { + const message = { + type: 'group', + groupId: 'group123', + context: { + traits: { + email: 'test@example.com', + customId: '123', + }, + }, + }; + const destination = { + Config: { + emersysCustomIdentifier: '100', + defaultContactList: 'list123', + fieldMapping: [ + { emersysProperty: '100', rudderProperty: 'customId' }, + { emersysProperty: '3', rudderProperty: 'email' }, + ], + }, + }; + const result = buildGroupPayload(message, destination); + expect(result).toEqual({ + eventType: 'group', + destinationPayload: { + payload: { + key_id: '100', + external_ids: ['123'], + }, + contactListId: 'group123', + }, + }); + }); + + it('should throw an InstrumentationError if emersysCustomIdentifier value is not present in payload', () => { + const message = { + type: 'group', + groupId: 'group123', + context: { + traits: { + email: 'test@example.com', + }, + }, + }; + const destination = { + Config: { + emersysCustomIdentifier: 'customId', + defaultContactList: 'list123', + fieldMapping: [ + { emersysProperty: 'customId', rudderProperty: 'customId' }, + { emersysProperty: 'email', rudderProperty: 'email' }, + ], + }, + }; + expect(() => { + buildGroupPayload(message, destination); + }).toThrow(InstrumentationError); + }); +}); diff --git a/src/constants/destinationCanonicalNames.js b/src/constants/destinationCanonicalNames.js index ee4f4f0b33..3cc05e70e3 100644 --- a/src/constants/destinationCanonicalNames.js +++ b/src/constants/destinationCanonicalNames.js @@ -166,6 +166,7 @@ const DestCanonicalNames = { ], koala: ['Koala', 'koala', 'KOALA'], bloomreach: ['Bloomreach', 'bloomreach', 'BLOOMREACH'], + emersys: ['EMERSYS', 'Emersys', 'emersys'], }; module.exports = { DestHandlerMap, DestCanonicalNames }; From 3522abff73fa50fd40ee21b36a4895761872a348 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Tue, 30 Apr 2024 13:09:16 +0530 Subject: [PATCH 05/26] fix: emersys unit test cases --- src/cdk/v2/destinations/emersys/config.js | 1 + src/cdk/v2/destinations/emersys/utils.js | 62 ++++----- src/cdk/v2/destinations/emersys/utils.test.js | 127 +++++++++++++++++- 3 files changed, 157 insertions(+), 33 deletions(-) diff --git a/src/cdk/v2/destinations/emersys/config.js b/src/cdk/v2/destinations/emersys/config.js index 604cac8cec..cbbf00f753 100644 --- a/src/cdk/v2/destinations/emersys/config.js +++ b/src/cdk/v2/destinations/emersys/config.js @@ -5,4 +5,5 @@ module.exports = { EMAIL_FIELD_ID: 3, OPT_IN_FILED_ID: 31, ALLOWED_OPT_IN_VALUES, + MAX_BATCH_SIZE_BYTES: 8000000, // 8 MB }; diff --git a/src/cdk/v2/destinations/emersys/utils.js b/src/cdk/v2/destinations/emersys/utils.js index f1aa5cdf2b..4dd515dc45 100644 --- a/src/cdk/v2/destinations/emersys/utils.js +++ b/src/cdk/v2/destinations/emersys/utils.js @@ -17,20 +17,21 @@ const { MAX_BATCH_SIZE, OPT_IN_FILED_ID, ALLOWED_OPT_IN_VALUES, + MAX_BATCH_SIZE_BYTES, } = require('./config'); -function base64Sha(str) { +const base64Sha = (str) => { const hexDigest = crypto.createHash('sha1').update(str).digest('hex'); return Buffer.from(hexDigest).toString('base64'); -} +}; -function getWsseHeader(user, secret) { +const getWsseHeader = (user, secret) => { const nonce = crypto.randomBytes(16).toString('hex'); const timestamp = new Date().toISOString(); const digest = base64Sha(nonce + timestamp + secret); return `UsernameToken Username="${user}", PasswordDigest="${digest}", Nonce="${nonce}", Created="${timestamp}"`; -} +}; const buildHeader = (destConfig) => { const { emersysUsername, emersysUserSecret } = destConfig; @@ -96,12 +97,12 @@ const buildIdentifyPayload = (message, destination) => { return { eventType: message.type, destinationPayload }; }; -function findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping) { +const findRudderPropertyByEmersysProperty = (emersysProperty, fieldMapping) => { // Use lodash to find the object where the emersysProperty matches the input const item = lodash.find(fieldMapping, { emersysProperty }); // Return the rudderProperty if the object is found, otherwise return null return item ? item.rudderProperty : null; -} +}; const buildGroupPayload = (message, destination) => { const { emersysCustomIdentifier, defaultContactList, fieldMapping } = destination.Config; @@ -149,16 +150,15 @@ const deduceEndPoint = (message, destConfig, batchGroupId = undefined) => { return endPoint; }; -function estimateJsonSize(obj) { - return new Blob([JSON.stringify(obj)]).size; -} +const estimateJsonSize = (obj) => new Blob([JSON.stringify(obj)]).size; -function createPayload(keyId, contacts, contactListId) { - return { key_id: keyId, contacts, contact_list_id: contactListId }; -} +const createSingleIdentifyPayload = (keyId, contacts, contactListId) => ({ + key_id: keyId, + contacts, + contact_list_id: contactListId, +}); -function ensureSizeConstraints(contacts) { - const MAX_SIZE_BYTES = 8000000; // 8 MB +const ensureSizeConstraints = (contacts) => { const chunks = []; let currentBatch = []; @@ -166,7 +166,7 @@ function ensureSizeConstraints(contacts) { // Start a new batch if adding the next contact exceeds size limits if ( currentBatch.length === 0 || - estimateJsonSize([...currentBatch, contact]) < MAX_SIZE_BYTES + estimateJsonSize([...currentBatch, contact]) < MAX_BATCH_SIZE_BYTES ) { currentBatch.push(contact); } else { @@ -181,9 +181,9 @@ function ensureSizeConstraints(contacts) { } return chunks; -} +}; -function createIdentifyBatches(events) { +const createIdentifyBatches = (events) => { const groupedIdentifyPayload = lodash.groupBy( events, (item) => @@ -204,13 +204,13 @@ function createIdentifyBatches(events) { // Include metadata for each chunk return finalChunks.map((contacts) => ({ - payload: createPayload(key_id, contacts, contact_list_id), - metadata: group.map((g) => g.metadata), // assuming metadata is needed per original event grouping + payload: createSingleIdentifyPayload(key_id, contacts, contact_list_id), + metadata: group.map((g) => g.metadata), })); }); -} +}; -function createGroupBatches(events) { +const createGroupBatches = (events) => { const grouped = lodash.groupBy( events, (item) => @@ -237,16 +237,15 @@ function createGroupBatches(events) { metadata: group.map((g) => g.metadata), // assuming metadata is needed per original event grouping })); }); -} -function formatIdentifyPayloadsWithEndpoint(combinedPayloads, endpointUrl = '') { - return combinedPayloads.map((payload) => ({ +}; +const formatIdentifyPayloadsWithEndpoint = (combinedPayloads, endpointUrl = '') => + combinedPayloads.map((payload) => ({ endpoint: endpointUrl, payload, })); -} -function buildBatchedRequest(batches, method, constants) { - return batches.map((batch) => ({ +const buildBatchedRequest = (batches, method, constants) => + batches.map((batch) => ({ batchedRequest: { body: { JSON: batch.payload, @@ -267,9 +266,8 @@ function buildBatchedRequest(batches, method, constants) { statusCode: 200, destination: constants.destination, })); -} -function batchResponseBuilder(successfulEvents) { +const batchResponseBuilder = (successfulEvents) => { const finaloutput = []; const groupedSuccessfulPayload = { identify: { @@ -334,7 +332,7 @@ function batchResponseBuilder(successfulEvents) { } return finaloutput; -} +}; module.exports = { buildIdentifyPayload, buildGroupPayload, @@ -345,4 +343,8 @@ module.exports = { getWsseHeader, findRudderPropertyByEmersysProperty, formatIdentifyPayloadsWithEndpoint, + createSingleIdentifyPayload, + createIdentifyBatches, + ensureSizeConstraints, + createGroupBatches, }; diff --git a/src/cdk/v2/destinations/emersys/utils.test.js b/src/cdk/v2/destinations/emersys/utils.test.js index d45171a82c..a113ced60d 100644 --- a/src/cdk/v2/destinations/emersys/utils.test.js +++ b/src/cdk/v2/destinations/emersys/utils.test.js @@ -1,14 +1,14 @@ +const lodash = require('lodash'); const { EVENT_TYPE } = require('rudder-transformer-cdk/build/constants'); const { buildIdentifyPayload, buildGroupPayload, - deduceEndPoint, - batchResponseBuilder, base64Sha, getWsseHeader, findRudderPropertyByEmersysProperty, - formatIdentifyPayloadsWithEndpoint, + createGroupBatches, } = require('./utils'); +const utils = require('./utils'); const crypto = require('crypto'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); @@ -288,3 +288,124 @@ describe('buildGroupPayload', () => { }).toThrow(InstrumentationError); }); }); + +describe('createGroupBatches', () => { + // Should group events by key_id and contactListId + it('should group events by key_id and contactListId when events are provided', () => { + // Arrange + const events = [ + { + message: { + body: { + JSON: { + destinationPayload: { + payload: { + key_id: 'key1', + external_ids: ['id1', 'id2'], + }, + contactListId: 'list1', + }, + }, + }, + }, + metadata: { jobId: 1, userId: 'u1' }, + }, + { + message: { + body: { + JSON: { + destinationPayload: { + payload: { + key_id: 'key2', + external_ids: ['id3', 'id4'], + }, + contactListId: 'list2', + }, + }, + }, + }, + metadata: { jobId: 2, userId: 'u2' }, + }, + { + message: { + body: { + JSON: { + destinationPayload: { + payload: { + key_id: 'key1', + external_ids: ['id5', 'id6'], + }, + contactListId: 'list1', + }, + }, + }, + }, + metadata: { jobId: 3, userId: 'u3' }, + }, + ]; + + // Act + const result = createGroupBatches(events); + + // Assert + expect(result).toEqual([ + { + endpoint: 'https://api.emarsys.net/api/v2/contactlist/list1/add', + payload: { + key_id: 'key1', + external_ids: ['id1', 'id2', 'id5', 'id6'], + }, + metadata: [ + { jobId: 1, userId: 'u1' }, + { jobId: 3, userId: 'u3' }, + ], + }, + { + endpoint: 'https://api.emarsys.net/api/v2/contactlist/list2/add', + payload: { + key_id: 'key2', + external_ids: ['id3', 'id4'], + }, + metadata: [{ jobId: 2, userId: 'u2' }], + }, + ]); + }); + + // Should return an empty array if no events are provided + it('should return an empty array when no events are provided', () => { + // Arrange + const events = []; + + // Act + const result = createGroupBatches(events); + + // Assert + expect(result).toEqual([]); + }); +}); + +describe('findRudderPropertyByEmersysProperty', () => { + // Returns the correct rudderProperty when given a valid emersysProperty and fieldMapping + it('should return the correct rudderProperty when given a valid emersysProperty and fieldMapping', () => { + const emersysProperty = 'email'; + const fieldMapping = [ + { emersysProperty: 'email', rudderProperty: 'email' }, + { emersysProperty: 'firstName', rudderProperty: 'firstName' }, + { emersysProperty: 'lastName', rudderProperty: 'lastName' }, + ]; + + const result = findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping); + + expect(result).toBe('email'); + }); + + // Returns null when given an empty fieldMapping + it('should return null when given an empty fieldMapping', () => { + const emersysProperty = 'email'; + const fieldMapping = []; + + const result = findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping); + + expect(result).toBeNull(); + }); +}); From f20b4032a860842f80a1ae58017491397f0640b3 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Tue, 30 Apr 2024 23:00:04 +0530 Subject: [PATCH 06/26] fix: adding support for track call --- .../v2/destinations/emersys/procWorkflow.yaml | 29 +- src/cdk/v2/destinations/emersys/utils.js | 85 +++- .../destinations/emersys/router/data.ts | 398 ++++++++++++++++++ 3 files changed, 500 insertions(+), 12 deletions(-) create mode 100644 test/integrations/destinations/emersys/router/data.ts diff --git a/src/cdk/v2/destinations/emersys/procWorkflow.yaml b/src/cdk/v2/destinations/emersys/procWorkflow.yaml index 7b4ba51529..e8c0a693de 100644 --- a/src/cdk/v2/destinations/emersys/procWorkflow.yaml +++ b/src/cdk/v2/destinations/emersys/procWorkflow.yaml @@ -39,19 +39,46 @@ steps: description: | Builds identify payload. ref: + condition: $.outputs.messageType === {{$.EventType.IDENTIFY}} template: | $.context.payload = $.buildIdentifyPayload(.message, .destination.Config,); - name: preparePayloadForGroup description: | Builds group payload. ref: https://dev.emarsys.com/docs/core-api-reference/1m0m70hy3tuov-add-contacts-to-a-contact-list + condition: $.outputs.messageType === {{$.EventType.GROUP}} template: | $.context.payload = $.buildGroupPayload(.message, .destination.Config,); + - name: preparePayloadForTrack + description: | + Builds track payload. + ref: https://dev.emarsys.com/docs/core-api-reference/fl0xx6rwfbwqb-trigger-an-external-event + condition: $.outputs.messageType === {{$.EventType.TRACK}} + template: | + const integrationObject = $.getIntegrationsObj(message, 'emersys'); + const emersysIdentifierId = integrationObject?.customIdentifierId || ^.destination.Config.emersysCustomIdentifier || $.EMAIL_FIELD_ID; + const payload = { + key_id: emersysIdentifierId , + external_id: $.deduceExternalIdValue(.message,emersysIdentifierId,.destination.Config.fieldMapping), + trigger_id: integrationObject.trigger_id, + data: properties.data, + attachment: Array.isArray(properties.attachment) + ? properties.attachment + : [properties.attachment], + event_time: $.getFieldValueFromMessage(message, 'timestamp'), + }; + $.context.payload = { + eventType: ^.message.type, + destinationPayload: { + payload: $.removeUndefinedAndNullValues(payload), + eventId: $.deduceEventId(.message,.destination.Config), + }, + }; - name: buildResponse template: | const response = $.defaultRequestConfig(); response.body.JSON = $.context.payload; - response.endpoint = $.deduceEndPoint(.message,.destination.Config); + response.endpoint = $.deduceEndPoint($.context.payload,.destination.Config); response.method = "POST"; response.headers = $.buildHeader(.destination.Config) response diff --git a/src/cdk/v2/destinations/emersys/utils.js b/src/cdk/v2/destinations/emersys/utils.js index 4dd515dc45..85404cdf0b 100644 --- a/src/cdk/v2/destinations/emersys/utils.js +++ b/src/cdk/v2/destinations/emersys/utils.js @@ -5,13 +5,18 @@ const crypto = require('crypto'); const { InstrumentationError, + ConfigurationError, isDefinedAndNotNullAndNotEmpty, removeUndefinedAndNullAndEmptyValues, removeUndefinedAndNullValues, isDefinedAndNotNull, + getHashFromArray, } = require('@rudderstack/integrations-lib'); -const { getValueFromMessage } = require('rudder-transformer-cdk/build/utils'); -const { getIntegrationsObj } = require('../../../../v0/util'); +const { + getValueFromMessage, + getFieldValueFromMessage, +} = require('rudder-transformer-cdk/build/utils'); +const { getIntegrationsObj, validateEventName } = require('../../../../v0/util'); const { EMAIL_FIELD_ID, MAX_BATCH_SIZE, @@ -104,16 +109,21 @@ const findRudderPropertyByEmersysProperty = (emersysProperty, fieldMapping) => { return item ? item.rudderProperty : null; }; -const buildGroupPayload = (message, destination) => { - const { emersysCustomIdentifier, defaultContactList, fieldMapping } = destination.Config; - const integrationObject = getIntegrationsObj(message, 'emersys'); - const emersysIdentifier = - integrationObject?.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; +const deduceExternalIdValue = (message, emersysIdentifier, fieldMapping) => { const configuredPayloadProperty = findRudderPropertyByEmersysProperty( emersysIdentifier, fieldMapping, ); const externalIdValue = getValueFromMessage(message.context.traits, configuredPayloadProperty); + return externalIdValue; +}; + +const buildGroupPayload = (message, destination) => { + const { emersysCustomIdentifier, defaultContactList, fieldMapping } = destination.Config; + const integrationObject = getIntegrationsObj(message, 'emersys'); + const emersysIdentifier = + integrationObject?.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; + const externalIdValue = deduceExternalIdValue(message, emersysIdentifier, fieldMapping); if (!isDefinedAndNotNull(externalIdValue)) { throw new InstrumentationError( `No value found in payload for contact custom identifier of id ${emersysIdentifier}`, @@ -132,18 +142,68 @@ const buildGroupPayload = (message, destination) => { }; }; -const deduceEndPoint = (message, destConfig, batchGroupId = undefined) => { +const deduceEventId = (message, destConfig) => { + let eventId; + const { eventsMapping } = destConfig; + const { event } = message; + validateEventName(event); + if (eventsMapping.length > 0) { + const keyMap = getHashFromArray(eventsMapping, 'from', 'to', false); + eventId = keyMap[event]; + } + if (!eventId) { + throw new ConfigurationError(`${event} is not mapped to any Emersys external event. Aborting`); + } +}; + +const buildTrackPayload = (message, destination) => { + let eventId; + const { emersysCustomIdentifier, eventsMapping } = destination.Config; + const { event, properties } = message; + + if (eventsMapping.length > 0) { + const keyMap = getHashFromArray(eventsMapping, 'from', 'to', false); + eventId = keyMap[event]; + } + const integrationObject = getIntegrationsObj(message, 'emersys'); + const emersysIdentifier = + integrationObject?.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; + const payload = { + key_id: emersysIdentifier, + external_id: 'test@example.com', + trigger_id: integrationObject.trigger_id, + data: properties.data, + attachment: Array.isArray(properties.attatchment) + ? properties.attatchment + : [properties.attatchment], + event_time: getFieldValueFromMessage(message, 'timestamp'), + }; + return { + eventType: message.type, + destinationPayload: { + payload: removeUndefinedAndNullValues(payload), + event: eventId, + }, + }; +}; + +const deduceEndPoint = (finalPayload) => { let endPoint; + let eventId; let contactListId; - const { type, groupId } = message; - switch (type) { + const { eventType, destinationPayload } = finalPayload; + switch (eventType) { case EVENT_TYPE.IDENTIFY: endPoint = 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1'; break; case EVENT_TYPE.GROUP: - contactListId = batchGroupId || groupId || destConfig.defaultContactList; + contactListId = destinationPayload.contactListId; endPoint = `https://api.emarsys.net/api/v2/contactlist/${contactListId}/add`; break; + case EVENT_TYPE.TRACK: + eventId = destinationPayload.eventId; + endPoint = `https://api.emarsys.net/api/v2/event/${eventId}/trigger`; + break; default: break; } @@ -347,4 +407,7 @@ module.exports = { createIdentifyBatches, ensureSizeConstraints, createGroupBatches, + buildTrackPayload, + deduceExternalIdValue, + deduceEventId, }; diff --git a/test/integrations/destinations/emersys/router/data.ts b/test/integrations/destinations/emersys/router/data.ts new file mode 100644 index 0000000000..e3e2526f38 --- /dev/null +++ b/test/integrations/destinations/emersys/router/data.ts @@ -0,0 +1,398 @@ +const config = { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '3', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + from: 'Email', + to: '3', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], +}; + +const commonDestination = { + ID: '12335', + Name: 'sample-destination', + DestinationDefinition: { + ID: '123', + Name: 'emersys', + DisplayName: 'Emersys', + Config: { + cdkV2Enabled: true, + }, + }, + WorkspaceID: '123', + Transformations: [], + Config: config, + Enabled: true, +}; + +export const data = [ + { + id: 'emersys-track-test-1', + name: 'emersys', + description: 'Track call : custom event calls with simple user properties and traits', + scenario: 'Business', + successCriteria: + 'event not respecting the internal mapping and as well as UI mapping should be considered as a custom event and should be sent as it is', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + type: 'identify', + sentAt: '2020-08-14T05: 30: 30.118Z', + channel: 'web', + context: { + source: 'test', + userAgent: 'chrome', + traits: { + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + email: 'abc@gmail.com', + phone: '+1234589947', + gender: 'non-binary', + db: '19950715', + lastname: 'Rudderlabs', + firstName: 'Test', + address: { + city: 'Kolkata', + state: 'WB', + zip: '700114', + country: 'IN', + }, + }, + device: { + advertisingId: 'abc123', + }, + library: { + name: 'rudder-sdk-ruby-sync', + version: '1.0.6', + }, + }, + messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + timestamp: '2024-02-10T12:16:07.251Z', + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + integrations: { + All: true, + }, + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 1, + secret: { + accessToken: 'dummyToken', + }, + }, + destination: commonDestination, + }, + { + message: { + type: 'identify', + sentAt: '2020-08-14T05: 30: 30.118Z', + channel: 'web', + context: { + source: 'test', + userAgent: 'chrome', + traits: { + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + email: 'abc@gmail.com', + phone: '+1234589947', + gender: 'non-binary', + db: '19950715', + lastname: 'Rudderlabs', + firstName: 'Test', + address: { + city: 'Kolkata', + state: 'WB', + zip: '700114', + country: 'IN', + }, + }, + device: { + advertisingId: 'abc123', + }, + library: { + name: 'rudder-sdk-ruby-sync', + version: '1.0.6', + }, + }, + messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + timestamp: '2024-02-10T12:16:07.251Z', + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + integrations: { + All: true, + }, + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 2, + secret: { + accessToken: 'dummyToken', + }, + }, + destination: commonDestination, + }, + { + message: { + type: 'identify', + sentAt: '2020-08-14T05: 30: 30.118Z', + channel: 'web', + context: { + source: 'test', + userAgent: 'chrome', + traits: { + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + phone: '+1234589947', + gender: 'non-binary', + db: '19950715', + lastname: 'Rudderlabs', + firstName: 'Test', + address: { + city: 'Kolkata', + state: 'WB', + zip: '700114', + country: 'IN', + }, + }, + device: { + advertisingId: 'abc123', + }, + library: { + name: 'rudder-sdk-ruby-sync', + version: '1.0.6', + }, + }, + messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + timestamp: '2024-02-10T12:16:07.251Z', + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + integrations: { + All: true, + }, + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 3, + secret: { + accessToken: 'dummyToken', + }, + }, + destination: commonDestination, + }, + ], + destType: 'emersys', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + metadata: [ + { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 3, + secret: { + accessToken: 'dummyToken', + }, + }, + ], + destination: { + ID: '12335', + Name: 'sample-destination', + DestinationDefinition: { + ID: '123', + Name: 'emersys', + DisplayName: 'emersys', + Config: { + cdkV2Enabled: true, + }, + }, + WorkspaceID: '123', + Transformations: [], + Config: config, + Enabled: true, + }, + batched: false, + statusCode: 400, + error: + '[emersys Conversion API] no matching user id found. Please provide at least one of the following: email, emersysFirstPartyAdsTrackingUUID, acxiomId, oracleMoatId', + statTags: { + destType: 'emersys', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'cdkV2', + module: 'destination', + }, + }, + { + batchedRequest: { + body: { + JSON: { + elements: [ + { + conversionHappenedAt: 1707567367251, + eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + conversionValue: { + currencyCode: 'USD', + amount: '50', + }, + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + userInfo: { + firstName: 'Test', + lastName: 'Rudderlabs', + }, + }, + conversion: 'urn:lla:llaPartnerConversion:1234567', + }, + { + conversionHappenedAt: 1707567367251, + eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + conversionValue: { + currencyCode: 'USD', + amount: '50', + }, + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + userInfo: { + firstName: 'Test', + lastName: 'Rudderlabs', + }, + }, + conversion: 'urn:lla:llaPartnerConversion:34567', + }, + { + conversionHappenedAt: 1707567367251, + eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + conversionValue: { + currencyCode: 'USD', + amount: '50', + }, + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + userInfo: { + firstName: 'Test', + lastName: 'Rudderlabs', + }, + }, + conversion: 'urn:lla:llaPartnerConversion:1234567', + }, + { + conversionHappenedAt: 1707567367251, + eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + conversionValue: { + currencyCode: 'USD', + amount: '50', + }, + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + userInfo: { + firstName: 'Test', + lastName: 'Rudderlabs', + }, + }, + conversion: 'urn:lla:llaPartnerConversion:34567', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emersys.com/rest/conversionEvents', + headers: { + 'Content-Type': 'application/json', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + 'emersys-Version': '202402', + Authorization: 'Bearer dummyToken', + }, + params: {}, + files: {}, + }, + metadata: [ + { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 1, + secret: { + accessToken: 'dummyToken', + }, + }, + { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 2, + secret: { + accessToken: 'dummyToken', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonDestination, + }, + ], + }, + }, + }, + }, +]; From f3fe891050f05b29cf7e1938ef2bfceacb144406 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Wed, 1 May 2024 23:36:44 +0530 Subject: [PATCH 07/26] fix: adding network hand;er support --- .../{emersys => emarsys}/config.js | 0 .../{emersys => emarsys}/procWorkflow.yaml | 0 .../{emersys => emarsys}/rtWorkflow.yaml | 0 .../{emersys => emarsys}/utils.js | 0 .../{emersys => emarsys}/utils.test.js | 0 src/services/destination/nativeIntegration.ts | 1 + src/v1/destinations/emarsys/networkHandler.js | 155 ++++++++++++++++++ 7 files changed, 156 insertions(+) rename src/cdk/v2/destinations/{emersys => emarsys}/config.js (100%) rename src/cdk/v2/destinations/{emersys => emarsys}/procWorkflow.yaml (100%) rename src/cdk/v2/destinations/{emersys => emarsys}/rtWorkflow.yaml (100%) rename src/cdk/v2/destinations/{emersys => emarsys}/utils.js (100%) rename src/cdk/v2/destinations/{emersys => emarsys}/utils.test.js (100%) create mode 100644 src/v1/destinations/emarsys/networkHandler.js diff --git a/src/cdk/v2/destinations/emersys/config.js b/src/cdk/v2/destinations/emarsys/config.js similarity index 100% rename from src/cdk/v2/destinations/emersys/config.js rename to src/cdk/v2/destinations/emarsys/config.js diff --git a/src/cdk/v2/destinations/emersys/procWorkflow.yaml b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml similarity index 100% rename from src/cdk/v2/destinations/emersys/procWorkflow.yaml rename to src/cdk/v2/destinations/emarsys/procWorkflow.yaml diff --git a/src/cdk/v2/destinations/emersys/rtWorkflow.yaml b/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml similarity index 100% rename from src/cdk/v2/destinations/emersys/rtWorkflow.yaml rename to src/cdk/v2/destinations/emarsys/rtWorkflow.yaml diff --git a/src/cdk/v2/destinations/emersys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js similarity index 100% rename from src/cdk/v2/destinations/emersys/utils.js rename to src/cdk/v2/destinations/emarsys/utils.js diff --git a/src/cdk/v2/destinations/emersys/utils.test.js b/src/cdk/v2/destinations/emarsys/utils.test.js similarity index 100% rename from src/cdk/v2/destinations/emersys/utils.test.js rename to src/cdk/v2/destinations/emarsys/utils.test.js diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 0bc9308fcd..8fd0f09857 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -221,6 +221,7 @@ export class NativeIntegrationDestinationService implements DestinationService { destinationResponse: processedProxyResponse, rudderJobMetadata, destType: destinationType, + destinationRequest: deliveryRequest, }; let responseProxy = networkHandler.responseHandler(responseParams); // Adaption Logic for V0 to V1 diff --git a/src/v1/destinations/emarsys/networkHandler.js b/src/v1/destinations/emarsys/networkHandler.js new file mode 100644 index 0000000000..f61b48de59 --- /dev/null +++ b/src/v1/destinations/emarsys/networkHandler.js @@ -0,0 +1,155 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); +const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); +const { isHttpStatusSuccess } = require('../../../v0/util/index'); + +const { + processAxiosResponse, + getDynamicErrorType, +} = require('../../../adapters/utils/networkUtils'); +const tags = require('../../../v0/util/tags'); + +function checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse) { + // Extract the errors from the destination response + const { errors } = destinationResponse.data; + const { key_id } = event; // Assuming the 'key_id' is constant as before + + // Find the first abortable case + const result = event.find((item) => { + if (typeof item === 'string') { + return errors[item]; // Check if the string is a key in errors + } + if (typeof item === 'object' && item[key_id]) { + return errors[item[key_id]]; // Check if the object's value under key_id is a key in errors + } + return false; // Continue if no condition is met + }); + + if (result) { + if (typeof result === 'string') { + // Handle case where result is a string key found in errors + return { + isAbortable: true, + errorMsg: errors[result][Object.keys(errors[result])[0]], + }; + } + if (typeof result === 'object') { + // Handle case where result is an object found in errors + const keyValue = result[key_id]; + return { + isAbortable: true, + errorMsg: errors[keyValue][Object.keys(errors[keyValue])[0]], + }; + } + } + + // If no match or abortable condition is found + return { + isAbortable: false, + errorMsg: '', + }; +} + +const responseHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata, destinationRequest } = responseParams; + const message = `[EMARSYS Response V1 Handler] - Request Processed Successfully`; + let responseWithIndividualEvents = []; + const { response, status } = destinationResponse; + + // even if a single event is unsuccessful, the entire batch will fail, we will filter that event out and retry others + if (!isHttpStatusSuccess(status)) { + const errorMessage = response.message || 'unknown error format'; + responseWithIndividualEvents = rudderJobMetadata.map((metadata) => ({ + statusCode: status, + metadata, + error: errorMessage, + })); + // if the status is 422, we need to parse the error message and construct the response array + // if (status === 422) { + // const destPartialStatus = constructPartialStatus(response?.message); + // // if the error message is not in the expected format, we will abort all of the events + // if (!destPartialStatus || lodash.isEmpty(destPartialStatus)) { + // throw new TransformerProxyError( + // `EMARSYS: Error transformer proxy v1 during EMARSYS response transformation. Error parsing error message`, + // status, + // { + // [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + // }, + // destinationResponse, + // getAuthErrCategoryFromStCode(status), + // responseWithIndividualEvents, + // ); + // } + // responseWithIndividualEvents = [...createResponseArray(rudderJobMetadata, destPartialStatus)]; + // return { + // status, + // message, + // destinationResponse, + // response: responseWithIndividualEvents, + // }; + // } + throw new TransformerProxyError( + `EMARSYS: Error transformer proxy v1 during EMARSYS response transformation. ${errorMessage}`, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + '', + responseWithIndividualEvents, + ); + } + + if (isHttpStatusSuccess(status)) { + // check for Partial Event failures and Successes + // eslint-disable-next-line @typescript-eslint/naming-convention + const { contacts, external_ids } = destinationRequest.body.JSON; + const finalData = contacts || external_ids; + finalData.forEach((event, idx) => { + const proxyOutput = { + statusCode: 200, + metadata: rudderJobMetadata[idx], + error: 'success', + }; + // update status of partial event if abortable + const { isAbortable, errorMsg } = checkIfEventIsAbortableAndExtractErrorMessage( + event, + destinationResponse, + ); + if (isAbortable) { + proxyOutput.statusCode = 400; + proxyOutput.error = errorMsg; + } + responseWithIndividualEvents.push(proxyOutput); + }); + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; + } + + // otherwise all events are successful + responseWithIndividualEvents = rudderJobMetadata.map((metadata) => ({ + statusCode: 200, + metadata, + error: 'success', + })); + + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; +}; + +function networkHandler() { + this.prepareProxy = prepareProxyRequest; + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.responseHandler = responseHandler; +} + +module.exports = { networkHandler }; From 0610d38ebe17c872c12b7c9fe1cec35ef5a6e557 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Thu, 2 May 2024 12:49:31 +0530 Subject: [PATCH 08/26] fix: track batch handled --- src/cdk/v2/destinations/emarsys/utils.js | 29 ++++++- src/cdk/v2/destinations/emarsys/utils.test.js | 81 ++++++++++++++++++- src/constants/destinationCanonicalNames.js | 2 +- src/v1/destinations/emarsys/networkHandler.js | 80 +++++------------- .../{emersys => emarsys}/router/data.ts | 22 ++--- 5 files changed, 137 insertions(+), 77 deletions(-) rename test/integrations/destinations/{emersys => emarsys}/router/data.ts (96%) diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index 85404cdf0b..1e7363e6ef 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -294,17 +294,23 @@ const createGroupBatches = (events) => { key_id: keyId, external_ids: chunk, }, - metadata: group.map((g) => g.metadata), // assuming metadata is needed per original event grouping + metadata: group.map((g) => g.metadata), })); }); }; + +const createTrackBatches = (events) => ({ + endpoint: events[0].message.endPoint, + payload: events[0].message.body.JSON.destinationPayload, + metadata: events[0].metadata, +}); const formatIdentifyPayloadsWithEndpoint = (combinedPayloads, endpointUrl = '') => combinedPayloads.map((payload) => ({ endpoint: endpointUrl, payload, })); -const buildBatchedRequest = (batches, method, constants) => +const buildBatchedRequest = (batches, method, constants, batchedStatus = true) => batches.map((batch) => ({ batchedRequest: { body: { @@ -322,7 +328,7 @@ const buildBatchedRequest = (batches, method, constants) => files: {}, }, metadata: batch.metadata, - batched: true, + batched: batchedStatus, statusCode: 200, destination: constants.destination, })); @@ -338,6 +344,10 @@ const batchResponseBuilder = (successfulEvents) => { method: 'POST', batches: [], }, + track: { + method: 'POST', + batches: [], + }, }; let batchesOfIdentifyEvents; if (successfulEvents.length === 0) { @@ -366,6 +376,9 @@ const batchResponseBuilder = (successfulEvents) => { case EVENT_TYPE.GROUP: groupedSuccessfulPayload.group.batches = createGroupBatches(eachEventGroup); break; + case EVENT_TYPE.TRACK: + groupedSuccessfulPayload.track.batches = createTrackBatches(eachEventGroup); + break; default: break; } @@ -391,6 +404,16 @@ const batchResponseBuilder = (successfulEvents) => { finaloutput.push(...groupBatches); } + if (groupedSuccessfulPayload.track) { + const trackBatches = buildBatchedRequest( + groupedSuccessfulPayload.track.batches, + groupedSuccessfulPayload.track.method, + constants, + false, + ); + finaloutput.push(...trackBatches); + } + return finaloutput; }; module.exports = { diff --git a/src/cdk/v2/destinations/emarsys/utils.test.js b/src/cdk/v2/destinations/emarsys/utils.test.js index a113ced60d..d9ce90097e 100644 --- a/src/cdk/v2/destinations/emarsys/utils.test.js +++ b/src/cdk/v2/destinations/emarsys/utils.test.js @@ -1,4 +1,3 @@ -const lodash = require('lodash'); const { EVENT_TYPE } = require('rudder-transformer-cdk/build/constants'); const { buildIdentifyPayload, @@ -8,7 +7,9 @@ const { findRudderPropertyByEmersysProperty, createGroupBatches, } = require('./utils'); -const utils = require('./utils'); +const { + checkIfEventIsAbortableAndExtractErrorMessage, +} = require('../../../../v1/destinations/emarsys/networkHandler'); const crypto = require('crypto'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); @@ -409,3 +410,79 @@ describe('findRudderPropertyByEmersysProperty', () => { expect(result).toBeNull(); }); }); + +describe('checkIfEventIsAbortableAndExtractErrorMessage', () => { + // Returns {isAbortable: false, errorMsg: ''} if event is neither a string nor an object with keyId. + it('should return {isAbortable: false, errorMsg: ""} when event is neither a string nor an object with keyId', () => { + const event = 123; + const destinationResponse = { + data: { + errors: { + errorKey: { + errorCode: 'errorMessage', + }, + }, + }, + }; + const keyId = 'keyId'; + + const result = checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId); + + expect(result).toEqual({ isAbortable: false, errorMsg: '' }); + }); + + // Returns {isAbortable: false, errorMsg: ''} if errors object is empty. + it('should return {isAbortable: false, errorMsg: ""} when errors object is empty', () => { + const event = 'event'; + const destinationResponse = { + data: { + errors: {}, + }, + }; + const keyId = 'keyId'; + + const result = checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId); + + expect(result).toEqual({ isAbortable: false, errorMsg: '' }); + }); + + // Returns {isAbortable: true, errorMsg} if event is a string and has a corresponding error in the errors object. + it('should return {isAbortable: true, errorMsg} when event is a string and has a corresponding error in the errors object', () => { + const event = 'event'; + const destinationResponse = { + data: { + errors: { + event: { + errorCode: 'errorMessage', + }, + }, + }, + }; + const keyId = 'keyId'; + + const result = checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId); + + expect(result).toEqual({ isAbortable: true, errorMsg: '{"errorCode":"errorMessage"}' }); + }); + + // Returns {isAbortable: true, errorMsg} if event is an object with keyId and has a corresponding error in the errors object. + it('should return {isAbortable: true, errorMsg} when event is an object with keyId and has a corresponding error in the errors object', () => { + const event = { + keyId: 'event', + }; + const destinationResponse = { + data: { + errors: { + event: { + errorCode: 'errorMessage', + }, + }, + }, + }; + const keyId = 'keyId'; + + const result = checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId); + + expect(result).toEqual({ isAbortable: true, errorMsg: '{"errorCode":"errorMessage"}' }); + }); +}); diff --git a/src/constants/destinationCanonicalNames.js b/src/constants/destinationCanonicalNames.js index 3cc05e70e3..b620146230 100644 --- a/src/constants/destinationCanonicalNames.js +++ b/src/constants/destinationCanonicalNames.js @@ -166,7 +166,7 @@ const DestCanonicalNames = { ], koala: ['Koala', 'koala', 'KOALA'], bloomreach: ['Bloomreach', 'bloomreach', 'BLOOMREACH'], - emersys: ['EMERSYS', 'Emersys', 'emersys'], + emersys: ['EMARSYS', 'Emarsys', 'emarsys'], }; module.exports = { DestHandlerMap, DestCanonicalNames }; diff --git a/src/v1/destinations/emarsys/networkHandler.js b/src/v1/destinations/emarsys/networkHandler.js index f61b48de59..641c89c0de 100644 --- a/src/v1/destinations/emarsys/networkHandler.js +++ b/src/v1/destinations/emarsys/networkHandler.js @@ -9,45 +9,28 @@ const { } = require('../../../adapters/utils/networkUtils'); const tags = require('../../../v0/util/tags'); -function checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse) { - // Extract the errors from the destination response +function checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId) { const { errors } = destinationResponse.data; - const { key_id } = event; // Assuming the 'key_id' is constant as before - // Find the first abortable case - const result = event.find((item) => { - if (typeof item === 'string') { - return errors[item]; // Check if the string is a key in errors - } - if (typeof item === 'object' && item[key_id]) { - return errors[item[key_id]]; // Check if the object's value under key_id is a key in errors - } - return false; // Continue if no condition is met - }); + // Determine if event is a string or an object, then fetch the corresponding key or value + let errorKey; + if (typeof event === 'string') { + errorKey = event; + } else if (typeof event === 'object' && event[keyId]) { + errorKey = event[keyId]; + } else { + return { isAbortable: false, errorMsg: '' }; // Early return if neither condition is met or if keyId is missing in the object + } - if (result) { - if (typeof result === 'string') { - // Handle case where result is a string key found in errors - return { - isAbortable: true, - errorMsg: errors[result][Object.keys(errors[result])[0]], - }; - } - if (typeof result === 'object') { - // Handle case where result is an object found in errors - const keyValue = result[key_id]; - return { - isAbortable: true, - errorMsg: errors[keyValue][Object.keys(errors[keyValue])[0]], - }; - } + // Check if this key has a corresponding error in the errors object + if (errors[errorKey]) { + // const errorCode = Object.keys(errors[errorKey])[0]; // Assume there is at least one error code + const errorMsg = JSON.stringify(errors[errorKey]); + return { isAbortable: true, errorMsg }; } - // If no match or abortable condition is found - return { - isAbortable: false, - errorMsg: '', - }; + // Return false and an empty error message if no error is found + return { isAbortable: false, errorMsg: '' }; } const responseHandler = (responseParams) => { @@ -64,30 +47,6 @@ const responseHandler = (responseParams) => { metadata, error: errorMessage, })); - // if the status is 422, we need to parse the error message and construct the response array - // if (status === 422) { - // const destPartialStatus = constructPartialStatus(response?.message); - // // if the error message is not in the expected format, we will abort all of the events - // if (!destPartialStatus || lodash.isEmpty(destPartialStatus)) { - // throw new TransformerProxyError( - // `EMARSYS: Error transformer proxy v1 during EMARSYS response transformation. Error parsing error message`, - // status, - // { - // [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), - // }, - // destinationResponse, - // getAuthErrCategoryFromStCode(status), - // responseWithIndividualEvents, - // ); - // } - // responseWithIndividualEvents = [...createResponseArray(rudderJobMetadata, destPartialStatus)]; - // return { - // status, - // message, - // destinationResponse, - // response: responseWithIndividualEvents, - // }; - // } throw new TransformerProxyError( `EMARSYS: Error transformer proxy v1 during EMARSYS response transformation. ${errorMessage}`, status, @@ -103,7 +62,7 @@ const responseHandler = (responseParams) => { if (isHttpStatusSuccess(status)) { // check for Partial Event failures and Successes // eslint-disable-next-line @typescript-eslint/naming-convention - const { contacts, external_ids } = destinationRequest.body.JSON; + const { contacts, external_ids, key_id } = destinationRequest.body.JSON; const finalData = contacts || external_ids; finalData.forEach((event, idx) => { const proxyOutput = { @@ -115,6 +74,7 @@ const responseHandler = (responseParams) => { const { isAbortable, errorMsg } = checkIfEventIsAbortableAndExtractErrorMessage( event, destinationResponse, + key_id, ); if (isAbortable) { proxyOutput.statusCode = 400; @@ -152,4 +112,4 @@ function networkHandler() { this.responseHandler = responseHandler; } -module.exports = { networkHandler }; +module.exports = { networkHandler, checkIfEventIsAbortableAndExtractErrorMessage }; diff --git a/test/integrations/destinations/emersys/router/data.ts b/test/integrations/destinations/emarsys/router/data.ts similarity index 96% rename from test/integrations/destinations/emersys/router/data.ts rename to test/integrations/destinations/emarsys/router/data.ts index e3e2526f38..f98058474f 100644 --- a/test/integrations/destinations/emersys/router/data.ts +++ b/test/integrations/destinations/emarsys/router/data.ts @@ -28,8 +28,8 @@ const commonDestination = { Name: 'sample-destination', DestinationDefinition: { ID: '123', - Name: 'emersys', - DisplayName: 'Emersys', + Name: 'emarsys', + DisplayName: 'Emarsys', Config: { cdkV2Enabled: true, }, @@ -42,8 +42,8 @@ const commonDestination = { export const data = [ { - id: 'emersys-track-test-1', - name: 'emersys', + id: 'emarsys-track-test-1', + name: 'emarsys', description: 'Track call : custom event calls with simple user properties and traits', scenario: 'Business', successCriteria: @@ -202,7 +202,7 @@ export const data = [ destination: commonDestination, }, ], - destType: 'emersys', + destType: 'emarsys', }, method: 'POST', }, @@ -229,8 +229,8 @@ export const data = [ Name: 'sample-destination', DestinationDefinition: { ID: '123', - Name: 'emersys', - DisplayName: 'emersys', + Name: 'emarsys', + DisplayName: 'emarsys', Config: { cdkV2Enabled: true, }, @@ -243,9 +243,9 @@ export const data = [ batched: false, statusCode: 400, error: - '[emersys Conversion API] no matching user id found. Please provide at least one of the following: email, emersysFirstPartyAdsTrackingUUID, acxiomId, oracleMoatId', + '[emarsys Conversion API] no matching user id found. Please provide at least one of the following: email, emersysFirstPartyAdsTrackingUUID, acxiomId, oracleMoatId', statTags: { - destType: 'emersys', + destType: 'emarsys', errorCategory: 'dataValidation', errorType: 'instrumentation', feature: 'router', @@ -355,12 +355,12 @@ export const data = [ version: '1', type: 'REST', method: 'POST', - endpoint: 'https://api.emersys.com/rest/conversionEvents', + endpoint: 'https://api.emarsys.com/rest/conversionEvents', headers: { 'Content-Type': 'application/json', 'X-RestLi-Method': 'BATCH_CREATE', 'X-Restli-Protocol-Version': '2.0.0', - 'emersys-Version': '202402', + 'emarsys-Version': '202402', Authorization: 'Bearer dummyToken', }, params: {}, From 04bf32c29f1017130567a4773974450edfe1614e Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Thu, 2 May 2024 14:58:25 +0530 Subject: [PATCH 09/26] fix: latest changes --- src/cdk/v2/destinations/emarsys/config.js | 17 ++++++++++++++++- src/cdk/v2/destinations/emarsys/rtWorkflow.yaml | 2 +- src/cdk/v2/destinations/emarsys/utils.js | 15 +-------------- .../destinations/emarsys/router/data.ts | 2 +- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/cdk/v2/destinations/emarsys/config.js b/src/cdk/v2/destinations/emarsys/config.js index cbbf00f753..83067c3cd3 100644 --- a/src/cdk/v2/destinations/emarsys/config.js +++ b/src/cdk/v2/destinations/emarsys/config.js @@ -1,9 +1,24 @@ const ALLOWED_OPT_IN_VALUES = ['1', '2', '']; +const groupedSuccessfulPayload = { + identify: { + method: 'PUT', + batches: [], + }, + group: { + method: 'POST', + batches: [], + }, + track: { + method: 'POST', + batches: [], + }, +}; module.exports = { MAX_BATCH_SIZE: 1000, EMAIL_FIELD_ID: 3, OPT_IN_FILED_ID: 31, ALLOWED_OPT_IN_VALUES, - MAX_BATCH_SIZE_BYTES: 8000000, // 8 MB + MAX_BATCH_SIZE_BYTES: 8000000, // 8 MB, + groupedSuccessfulPayload, }; diff --git a/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml b/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml index dda322e45e..84e31d54a2 100644 --- a/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml +++ b/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml @@ -1,6 +1,5 @@ bindings: - path: ./utils - - path: ./config - name: handleRtTfSingleEventError path: ../../../../v0/util/index @@ -31,6 +30,7 @@ steps: - name: batchSuccessfulEvents description: Batches the successfulEvents + debug: true template: | $.batchResponseBuilder($.outputs.successfulEvents); diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index 1e7363e6ef..71fe0bb9f2 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -23,6 +23,7 @@ const { OPT_IN_FILED_ID, ALLOWED_OPT_IN_VALUES, MAX_BATCH_SIZE_BYTES, + groupedSuccessfulPayload, } = require('./config'); const base64Sha = (str) => { @@ -335,20 +336,6 @@ const buildBatchedRequest = (batches, method, constants, batchedStatus = true) = const batchResponseBuilder = (successfulEvents) => { const finaloutput = []; - const groupedSuccessfulPayload = { - identify: { - method: 'PUT', - batches: [], - }, - group: { - method: 'POST', - batches: [], - }, - track: { - method: 'POST', - batches: [], - }, - }; let batchesOfIdentifyEvents; if (successfulEvents.length === 0) { return []; diff --git a/test/integrations/destinations/emarsys/router/data.ts b/test/integrations/destinations/emarsys/router/data.ts index f98058474f..fe6e594738 100644 --- a/test/integrations/destinations/emarsys/router/data.ts +++ b/test/integrations/destinations/emarsys/router/data.ts @@ -243,7 +243,7 @@ export const data = [ batched: false, statusCode: 400, error: - '[emarsys Conversion API] no matching user id found. Please provide at least one of the following: email, emersysFirstPartyAdsTrackingUUID, acxiomId, oracleMoatId', + '[emarsys] no matching user id found. Please provide at least one of the following: email, emersysFirstPartyAdsTrackingUUID, acxiomId, oracleMoatId', statTags: { destType: 'emarsys', errorCategory: 'dataValidation', From 608469b5a351fbd94aaca2c7b7a87a65e4486376 Mon Sep 17 00:00:00 2001 From: Dilip Kola Date: Thu, 2 May 2024 15:52:45 +0530 Subject: [PATCH 10/26] refactor: remove cdk v1 imports --- src/cdk/v2/destinations/emarsys/utils.js | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index 71fe0bb9f2..49ea541213 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -1,8 +1,5 @@ -import { EVENT_TYPE } from 'rudder-transformer-cdk/build/constants'; - const lodash = require('lodash'); const crypto = require('crypto'); - const { InstrumentationError, ConfigurationError, @@ -13,10 +10,11 @@ const { getHashFromArray, } = require('@rudderstack/integrations-lib'); const { + getIntegrationsObj, + validateEventName, getValueFromMessage, getFieldValueFromMessage, -} = require('rudder-transformer-cdk/build/utils'); -const { getIntegrationsObj, validateEventName } = require('../../../../v0/util'); +} = require('../../../../v0/util'); const { EMAIL_FIELD_ID, MAX_BATCH_SIZE, @@ -25,6 +23,8 @@ const { MAX_BATCH_SIZE_BYTES, groupedSuccessfulPayload, } = require('./config'); +const { SUPPORTED_EVENT_TYPE } = require('../the_trade_desk_real_time_conversions/config'); +const { EventType } = require('../../../../constants'); const base64Sha = (str) => { const hexDigest = crypto.createHash('sha1').update(str).digest('hex'); @@ -194,14 +194,14 @@ const deduceEndPoint = (finalPayload) => { let contactListId; const { eventType, destinationPayload } = finalPayload; switch (eventType) { - case EVENT_TYPE.IDENTIFY: + case EventType.IDENTIFY: endPoint = 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1'; break; - case EVENT_TYPE.GROUP: + case SUPPORTED_EVENT_TYPE.GROUP: contactListId = destinationPayload.contactListId; endPoint = `https://api.emarsys.net/api/v2/contactlist/${contactListId}/add`; break; - case EVENT_TYPE.TRACK: + case EventType.TRACK: eventId = destinationPayload.eventId; endPoint = `https://api.emarsys.net/api/v2/event/${eventId}/trigger`; break; @@ -353,17 +353,17 @@ const batchResponseBuilder = (successfulEvents) => { ); Object.keys(typedEventGroups).forEach((eachEventGroup) => { switch (eachEventGroup) { - case EVENT_TYPE.IDENTIFY: + case EventType.IDENTIFY: batchesOfIdentifyEvents = createIdentifyBatches(eachEventGroup); groupedSuccessfulPayload.identify.batches = formatIdentifyPayloadsWithEndpoint( batchesOfIdentifyEvents, 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', ); break; - case EVENT_TYPE.GROUP: + case EventType.GROUP: groupedSuccessfulPayload.group.batches = createGroupBatches(eachEventGroup); break; - case EVENT_TYPE.TRACK: + case EventType.TRACK: groupedSuccessfulPayload.track.batches = createTrackBatches(eachEventGroup); break; default: @@ -403,6 +403,7 @@ const batchResponseBuilder = (successfulEvents) => { return finaloutput; }; + module.exports = { buildIdentifyPayload, buildGroupPayload, From 68ab695ad903d086eba1f7383addf71b2cf919b4 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Thu, 2 May 2024 21:55:53 +0530 Subject: [PATCH 11/26] fix: identify batching final --- .../v2/destinations/emarsys/procWorkflow.yaml | 2 +- .../v2/destinations/emarsys/rtWorkflow.yaml | 6 +- src/cdk/v2/destinations/emarsys/utils.js | 33 +- src/cdk/v2/destinations/emarsys/utils.test.js | 64 ++- src/constants/destinationCanonicalNames.js | 2 +- src/v0/destinations/emarsys/deleteUsers.js | 82 ++++ .../destinations/emarsys/router/data.ts | 390 ++++++++---------- 7 files changed, 315 insertions(+), 264 deletions(-) create mode 100644 src/v0/destinations/emarsys/deleteUsers.js diff --git a/src/cdk/v2/destinations/emarsys/procWorkflow.yaml b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml index e8c0a693de..d8a84959b3 100644 --- a/src/cdk/v2/destinations/emarsys/procWorkflow.yaml +++ b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml @@ -55,7 +55,7 @@ steps: ref: https://dev.emarsys.com/docs/core-api-reference/fl0xx6rwfbwqb-trigger-an-external-event condition: $.outputs.messageType === {{$.EventType.TRACK}} template: | - const integrationObject = $.getIntegrationsObj(message, 'emersys'); + const integrationObject = $.getIntegrationsObj(^.message, 'emersys'); const emersysIdentifierId = integrationObject?.customIdentifierId || ^.destination.Config.emersysCustomIdentifier || $.EMAIL_FIELD_ID; const payload = { key_id: emersysIdentifierId , diff --git a/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml b/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml index 84e31d54a2..36df10bb43 100644 --- a/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml +++ b/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml @@ -30,10 +30,10 @@ steps: - name: batchSuccessfulEvents description: Batches the successfulEvents - debug: true template: | - $.batchResponseBuilder($.outputs.successfulEvents); + $.context.batchedPayload = $.batchResponseBuilder($.outputs.successfulEvents); + console.log("batchedPayload",JSON.stringify($.context.batchedPayload)); - name: finalPayload template: | - [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents] + [...$.outputs.failedEvents, ...$.context.batchedPayload] diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index 49ea541213..9d7faff01b 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -48,13 +48,13 @@ const buildHeader = (destConfig) => { }; }; -const buildIdentifyPayload = (message, destination) => { +const buildIdentifyPayload = (message, destConfig) => { let destinationPayload; const { fieldMapping, emersysCustomIdentifier, discardEmptyProperties, defaultContactList } = - destination.Config; + destConfig; const payload = {}; - const integrationObject = getIntegrationsObj(message, 'emersys'); + const integrationObject = getIntegrationsObj(message, 'emarsys'); const finalContactList = integrationObject?.contactListId || defaultContactList; if (!isDefinedAndNotNullAndNotEmpty(finalContactList)) { @@ -65,9 +65,10 @@ const buildIdentifyPayload = (message, destination) => { if (fieldMapping) { fieldMapping.forEach((trait) => { const { rudderProperty, emersysProperty } = trait; - const value = - getValueFromMessage(message.traits, rudderProperty) || - getValueFromMessage(message.context.traits, rudderProperty); + const value = getValueFromMessage(message, [ + `traits.${rudderProperty}`, + `context.traits.${rudderProperty}`, + ]); if (value) { payload[emersysProperty] = value; } @@ -99,7 +100,6 @@ const buildIdentifyPayload = (message, destination) => { 'Either configured custom contact identifier value or default identifier email value is missing', ); } - return { eventType: message.type, destinationPayload }; }; @@ -248,17 +248,16 @@ const createIdentifyBatches = (events) => { const groupedIdentifyPayload = lodash.groupBy( events, (item) => - `${item.message.body.JSON.destinationPayload.key_id}-${item.message.body.JSON.destinationPayload.contact_list_id}`, + `${item.message[0].body.JSON.destinationPayload.key_id}-${item.message[0].body.JSON.destinationPayload.contact_list_id}`, ); - return lodash.flatMap(groupedIdentifyPayload, (group) => { - const firstItem = group[0].message.body.JSON.destinationPayload; + const firstItem = group[0].message[0].body.JSON.destinationPayload; // eslint-disable-next-line @typescript-eslint/naming-convention const { key_id, contact_list_id } = firstItem; const allContacts = lodash.flatMap( group, - (item) => item.message.body.JSON.destinationPayload.contacts, + (item) => item.message[0].body.JSON.destinationPayload.contacts, ); const initialChunks = lodash.chunk(allContacts, MAX_BATCH_SIZE); const finalChunks = lodash.flatMap(initialChunks, ensureSizeConstraints); @@ -306,9 +305,10 @@ const createTrackBatches = (events) => ({ metadata: events[0].metadata, }); const formatIdentifyPayloadsWithEndpoint = (combinedPayloads, endpointUrl = '') => - combinedPayloads.map((payload) => ({ + combinedPayloads.map((singleCombinedPayload) => ({ endpoint: endpointUrl, - payload, + payload: singleCombinedPayload.payload, + metadata: singleCombinedPayload.metadata, })); const buildBatchedRequest = (batches, method, constants, batchedStatus = true) => @@ -349,12 +349,12 @@ const batchResponseBuilder = (successfulEvents) => { const typedEventGroups = lodash.groupBy( successfulEvents, - (event) => event.message.body.JSON.eventType, + (event) => event.message[0].body.JSON.eventType, ); Object.keys(typedEventGroups).forEach((eachEventGroup) => { switch (eachEventGroup) { case EventType.IDENTIFY: - batchesOfIdentifyEvents = createIdentifyBatches(eachEventGroup); + batchesOfIdentifyEvents = createIdentifyBatches(typedEventGroups[eachEventGroup]); groupedSuccessfulPayload.identify.batches = formatIdentifyPayloadsWithEndpoint( batchesOfIdentifyEvents, 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', @@ -371,6 +371,7 @@ const batchResponseBuilder = (successfulEvents) => { } return groupedSuccessfulPayload; }); + console.log('groupedSuccessfulPayload', JSON.stringify(groupedSuccessfulPayload)); // Process each identify batch if (groupedSuccessfulPayload.identify) { const identifyBatches = buildBatchedRequest( @@ -400,7 +401,7 @@ const batchResponseBuilder = (successfulEvents) => { ); finaloutput.push(...trackBatches); } - + console.log('FINAL', JSON.stringify(finaloutput)); return finaloutput; }; diff --git a/src/cdk/v2/destinations/emarsys/utils.test.js b/src/cdk/v2/destinations/emarsys/utils.test.js index d9ce90097e..15147d9cdd 100644 --- a/src/cdk/v2/destinations/emarsys/utils.test.js +++ b/src/cdk/v2/destinations/emarsys/utils.test.js @@ -66,15 +66,13 @@ describe('buildIdentifyPayload', () => { }, }; const destination = { - Config: { - fieldMapping: [ - { rudderProperty: 'firstName', emersysProperty: '1' }, - { rudderProperty: 'lastName', emersysProperty: '2' }, - { rudderProperty: 'email', emersysProperty: '3' }, - { rudderProperty: 'optin', emersysProperty: '31' }, - ], - defaultContactList: 'dummyContactList', - }, + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + defaultContactList: 'dummyContactList', }; const expectedPayload = { contact_list_id: 'dummyContactList', @@ -106,15 +104,13 @@ describe('buildIdentifyPayload', () => { }, }; const destination = { - Config: { - fieldMapping: [ - { rudderProperty: 'firstName', emersysProperty: '1' }, - { rudderProperty: 'lastName', emersysProperty: '2' }, - { rudderProperty: 'email', emersysProperty: '3' }, - { rudderProperty: 'optin', emersysProperty: '31' }, - ], - defaultContactList: 'dummyList', - }, + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + defaultContactList: 'dummyList', }; expect(() => { buildIdentifyPayload(message, destination); @@ -132,14 +128,12 @@ describe('buildIdentifyPayload', () => { }, }; const destination = { - Config: { - fieldMapping: [ - { rudderProperty: 'firstName', emersysProperty: '1' }, - { rudderProperty: 'lastName', emersysProperty: '2' }, - { rudderProperty: 'email', emersysProperty: '3' }, - { rudderProperty: 'optin', emersysProperty: '31' }, - ], - }, + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], }; expect(() => { buildIdentifyPayload(message, destination); @@ -158,22 +152,20 @@ describe('buildIdentifyPayload', () => { optin: 1, }, integrations: { - EMERSYS: { + EMARSYS: { customIdentifierId: 1, contactListId: 'objectListId', }, }, }; const destination = { - Config: { - fieldMapping: [ - { rudderProperty: 'firstName', emersysProperty: '1' }, - { rudderProperty: 'lastName', emersysProperty: '2' }, - { rudderProperty: 'email', emersysProperty: '3' }, - { rudderProperty: 'optin', emersysProperty: '31' }, - ], - defaultContactList: 'dummyContactList', - }, + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + defaultContactList: 'dummyContactList', }; const expectedPayload = { contact_list_id: 'objectListId', diff --git a/src/constants/destinationCanonicalNames.js b/src/constants/destinationCanonicalNames.js index b620146230..19136eab59 100644 --- a/src/constants/destinationCanonicalNames.js +++ b/src/constants/destinationCanonicalNames.js @@ -166,7 +166,7 @@ const DestCanonicalNames = { ], koala: ['Koala', 'koala', 'KOALA'], bloomreach: ['Bloomreach', 'bloomreach', 'BLOOMREACH'], - emersys: ['EMARSYS', 'Emarsys', 'emarsys'], + emarsys: ['EMARSYS', 'Emarsys', 'emarsys'], }; module.exports = { DestHandlerMap, DestCanonicalNames }; diff --git a/src/v0/destinations/emarsys/deleteUsers.js b/src/v0/destinations/emarsys/deleteUsers.js new file mode 100644 index 0000000000..01044adcd1 --- /dev/null +++ b/src/v0/destinations/emarsys/deleteUsers.js @@ -0,0 +1,82 @@ +const { NetworkError, ConfigurationError } = require('@rudderstack/integrations-lib'); +const { httpPOST } = require('../../../adapters/network'); +const { + processAxiosResponse, + getDynamicErrorType, +} = require('../../../adapters/utils/networkUtils'); +const { isHttpStatusSuccess } = require('../../util'); +const { executeCommonValidations } = require('../../util/regulation-api'); +const tags = require('../../util/tags'); +const { getUserIdBatches } = require('../../util/deleteUserUtils'); +const { JSON_MIME_TYPE } = require('../../util/constant'); + +/** + * This function will help to delete the users one by one from the userAttributes array. + * @param {*} userAttributes Array of objects with userId, email and phone + * @param {*} config Destination.Config provided in dashboard + * @returns + */ +const userDeletionHandler = async (userAttributes, config) => { + const { apiKey } = config; + + if (!apiKey) { + throw new ConfigurationError('Api Key is required for user deletion'); + } + + const endpoint = 'https://api.sprig.com/v2/purge/visitors'; + const headers = { + Accept: JSON_MIME_TYPE, + 'Content-Type': JSON_MIME_TYPE, + Authorization: `API-Key ${apiKey}`, + }; + /** + * userIdBatches = [[u1,u2,u3,...batchSize],[u1,u2,u3,...batchSize]...] + * Ref doc : https://docs.sprig.com/reference/post-v2-purge-visitors-1 + */ + const userIdBatches = getUserIdBatches(userAttributes, 100); + // Note: we will only get 400 status code when no user deletion is present for given userIds so we will not throw error in that case + // eslint-disable-next-line no-restricted-syntax + for (const curBatch of userIdBatches) { + // eslint-disable-next-line no-await-in-loop + const deletionResponse = await httpPOST( + endpoint, + { + userIds: curBatch, + }, + { + headers, + }, + { + destType: 'sprig', + feature: 'deleteUsers', + endpointPath: '/purge/visitors', + requestMethod: 'POST', + module: 'deletion', + }, + ); + const handledDelResponse = processAxiosResponse(deletionResponse); + if (!isHttpStatusSuccess(handledDelResponse.status) && handledDelResponse.status !== 400) { + throw new NetworkError( + 'User deletion request failed', + handledDelResponse.status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(handledDelResponse.status), + [tags.TAG_NAMES.STATUS]: handledDelResponse.status, + }, + handledDelResponse, + ); + } + } + + return { + statusCode: 200, + status: 'successful', + }; +}; +const processDeleteUsers = async (event) => { + const { userAttributes, config } = event; + executeCommonValidations(userAttributes); + const resp = await userDeletionHandler(userAttributes, config); + return resp; +}; +module.exports = { processDeleteUsers }; diff --git a/test/integrations/destinations/emarsys/router/data.ts b/test/integrations/destinations/emarsys/router/data.ts index fe6e594738..32f5782474 100644 --- a/test/integrations/destinations/emarsys/router/data.ts +++ b/test/integrations/destinations/emarsys/router/data.ts @@ -12,8 +12,16 @@ const config = { ], fieldMapping: [ { - from: 'Email', - to: '3', + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'firstName', + emersysProperty: '1', + }, + { + rudderProperty: 'lastName', + emersysProperty: '2', }, ], oneTrustCookieCategories: [ @@ -57,41 +65,25 @@ export const data = [ input: [ { message: { - type: 'identify', - sentAt: '2020-08-14T05: 30: 30.118Z', channel: 'web', context: { - source: 'test', - userAgent: 'chrome', traits: { - anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', - email: 'abc@gmail.com', - phone: '+1234589947', - gender: 'non-binary', - db: '19950715', - lastname: 'Rudderlabs', - firstName: 'Test', - address: { - city: 'Kolkata', - state: 'WB', - zip: '700114', - country: 'IN', - }, - }, - device: { - advertisingId: 'abc123', - }, - library: { - name: 'rudder-sdk-ruby-sync', - version: '1.0.6', + email: 'testone@gmail.com', + firstName: 'test', + lastName: 'one', }, }, - messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', - timestamp: '2024-02-10T12:16:07.251Z', - anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + event: 'product list viewed', + userId: 'testuserId1', integrations: { All: true, }, + sentAt: '2019-10-14T09:03:22.563Z', }, metadata: { sourceType: '', @@ -106,41 +98,25 @@ export const data = [ }, { message: { - type: 'identify', - sentAt: '2020-08-14T05: 30: 30.118Z', channel: 'web', context: { - source: 'test', - userAgent: 'chrome', traits: { - anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', - email: 'abc@gmail.com', - phone: '+1234589947', - gender: 'non-binary', - db: '19950715', - lastname: 'Rudderlabs', - firstName: 'Test', - address: { - city: 'Kolkata', - state: 'WB', - zip: '700114', - country: 'IN', - }, - }, - device: { - advertisingId: 'abc123', - }, - library: { - name: 'rudder-sdk-ruby-sync', - version: '1.0.6', + email: 'testtwo@gmail.com', + firstName: 'test', + lastName: 'one', }, }, - messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', - timestamp: '2024-02-10T12:16:07.251Z', - anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + event: 'product list viewed', + userId: 'testuserId1', integrations: { All: true, }, + sentAt: '2019-10-14T09:03:22.563Z', }, metadata: { sourceType: '', @@ -155,46 +131,72 @@ export const data = [ }, { message: { - type: 'identify', - sentAt: '2020-08-14T05: 30: 30.118Z', channel: 'web', context: { - source: 'test', - userAgent: 'chrome', traits: { - anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', - phone: '+1234589947', - gender: 'non-binary', - db: '19950715', - lastname: 'Rudderlabs', - firstName: 'Test', - address: { - city: 'Kolkata', - state: 'WB', - zip: '700114', - country: 'IN', - }, + email: 'testtwo@gmail.com', + firstName: 'test', + lastName: 'one', }, - device: { - advertisingId: 'abc123', + }, + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + event: 'product list viewed', + userId: 'testuserId1', + integrations: { + All: true, + EMARSYS: { + contactListId: 'dummy2', + customIdentifierId: '1', }, - library: { - name: 'rudder-sdk-ruby-sync', - version: '1.0.6', + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 3, + secret: { + accessToken: 'dummyToken', + }, + }, + destination: commonDestination, + }, + { + message: { + channel: 'web', + context: { + traits: { + email: 'testtwo@gmail.com', + firstName: 'test', + lastName: 'one', }, }, - messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', - timestamp: '2024-02-10T12:16:07.251Z', - anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + event: 'product list viewed', + userId: 'testuserId1', integrations: { All: true, + EMARSYS: { + contactListId: 'dummy2', + customIdentifierId: '2', + }, }, + sentAt: '2019-10-14T09:03:22.563Z', }, metadata: { sourceType: '', destinationType: '', namespace: '', - jobId: 3, + jobId: 4, secret: { accessToken: 'dummyToken', }, @@ -215,138 +217,41 @@ export const data = [ { metadata: [ { - sourceType: '', destinationType: '', + jobId: 1, namespace: '', - jobId: 3, secret: { accessToken: 'dummyToken', }, + sourceType: '', }, - ], - destination: { - ID: '12335', - Name: 'sample-destination', - DestinationDefinition: { - ID: '123', - Name: 'emarsys', - DisplayName: 'emarsys', - Config: { - cdkV2Enabled: true, + { + destinationType: '', + jobId: 2, + namespace: '', + secret: { + accessToken: 'dummyToken', }, + sourceType: '', }, - WorkspaceID: '123', - Transformations: [], - Config: config, - Enabled: true, - }, - batched: false, - statusCode: 400, - error: - '[emarsys] no matching user id found. Please provide at least one of the following: email, emersysFirstPartyAdsTrackingUUID, acxiomId, oracleMoatId', - statTags: { - destType: 'emarsys', - errorCategory: 'dataValidation', - errorType: 'instrumentation', - feature: 'router', - implementation: 'cdkV2', - module: 'destination', - }, - }, - { + ], batchedRequest: { body: { JSON: { - elements: [ - { - conversionHappenedAt: 1707567367251, - eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', - conversionValue: { - currencyCode: 'USD', - amount: '50', - }, - user: { - userIds: [ - { - idType: 'SHA256_EMAIL', - idValue: - '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', - }, - ], - userInfo: { - firstName: 'Test', - lastName: 'Rudderlabs', - }, - }, - conversion: 'urn:lla:llaPartnerConversion:1234567', - }, - { - conversionHappenedAt: 1707567367251, - eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', - conversionValue: { - currencyCode: 'USD', - amount: '50', - }, - user: { - userIds: [ - { - idType: 'SHA256_EMAIL', - idValue: - '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', - }, - ], - userInfo: { - firstName: 'Test', - lastName: 'Rudderlabs', - }, - }, - conversion: 'urn:lla:llaPartnerConversion:34567', - }, + key_id: '3', + contacts: [ { - conversionHappenedAt: 1707567367251, - eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', - conversionValue: { - currencyCode: 'USD', - amount: '50', - }, - user: { - userIds: [ - { - idType: 'SHA256_EMAIL', - idValue: - '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', - }, - ], - userInfo: { - firstName: 'Test', - lastName: 'Rudderlabs', - }, - }, - conversion: 'urn:lla:llaPartnerConversion:1234567', + '3': 'testone@gmail.com', + '1': 'test', + '2': 'one', }, { - conversionHappenedAt: 1707567367251, - eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', - conversionValue: { - currencyCode: 'USD', - amount: '50', - }, - user: { - userIds: [ - { - idType: 'SHA256_EMAIL', - idValue: - '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', - }, - ], - userInfo: { - firstName: 'Test', - lastName: 'Rudderlabs', - }, - }, - conversion: 'urn:lla:llaPartnerConversion:34567', + '3': 'testtwo@gmail.com', + '1': 'test', + '2': 'one', }, ], + contact_list_id: 'dummy', }, JSON_ARRAY: {}, XML: {}, @@ -354,38 +259,109 @@ export const data = [ }, version: '1', type: 'REST', - method: 'POST', - endpoint: 'https://api.emarsys.com/rest/conversionEvents', + method: 'PUT', + endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', headers: { 'Content-Type': 'application/json', - 'X-RestLi-Method': 'BATCH_CREATE', - 'X-Restli-Protocol-Version': '2.0.0', - 'emarsys-Version': '202402', - Authorization: 'Bearer dummyToken', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="OWM2ODlmYjZiMDA0YTQwZjc1NjkyOWFiZTA1MTQ0ZmUwOGYyYWQ2NA==", Nonce="8c02af01eb527f450340bb82ebd40dde", Created="2024-05-02T15:41:20.529Z"', }, params: {}, files: {}, }, + batched: true, + statusCode: 200, + destination: commonDestination, + }, + { metadata: [ { - sourceType: '', destinationType: '', + jobId: 3, namespace: '', - jobId: 1, secret: { accessToken: 'dummyToken', }, + sourceType: '', }, + ], + batchedRequest: { + body: { + JSON: { + key_id: '1', + contacts: [ + { + '3': 'testtwo@gmail.com', + '1': 'test', + '2': 'one', + }, + ], + contact_list_id: 'dummy2', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'PUT', + endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="OWM2ODlmYjZiMDA0YTQwZjc1NjkyOWFiZTA1MTQ0ZmUwOGYyYWQ2NA==", Nonce="8c02af01eb527f450340bb82ebd40dde", Created="2024-05-02T15:41:20.529Z"', + }, + params: {}, + files: {}, + }, + batched: true, + statusCode: 200, + destination: commonDestination, + }, + { + metadata: [ { - sourceType: '', destinationType: '', + jobId: 4, namespace: '', - jobId: 2, secret: { accessToken: 'dummyToken', }, + sourceType: '', }, ], + batchedRequest: { + body: { + JSON: { + key_id: '2', + contacts: [ + { + '3': 'testtwo@gmail.com', + '1': 'test', + '2': 'one', + }, + ], + contact_list_id: 'dummy2', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'PUT', + endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="OWM2ODlmYjZiMDA0YTQwZjc1NjkyOWFiZTA1MTQ0ZmUwOGYyYWQ2NA==", Nonce="8c02af01eb527f450340bb82ebd40dde", Created="2024-05-02T15:41:20.529Z"', + }, + params: {}, + files: {}, + }, batched: true, statusCode: 200, destination: commonDestination, From 54ed0b20d76bec3acb8125d2c526c8f647e66e24 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Fri, 3 May 2024 22:54:08 +0530 Subject: [PATCH 12/26] fix: all batching final --- .../v2/destinations/emarsys/procWorkflow.yaml | 21 +- .../v2/destinations/emarsys/rtWorkflow.yaml | 1 - src/cdk/v2/destinations/emarsys/utils.js | 213 +++++++---- .../destinations/emarsys/router/data.ts | 360 +++++++++++++++--- 4 files changed, 474 insertions(+), 121 deletions(-) diff --git a/src/cdk/v2/destinations/emarsys/procWorkflow.yaml b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml index d8a84959b3..ee7f78b743 100644 --- a/src/cdk/v2/destinations/emarsys/procWorkflow.yaml +++ b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml @@ -5,8 +5,14 @@ bindings: exportAll: true - name: removeUndefinedValues path: ../../../../v0/util + - name: removeUndefinedAndNullValues + path: ../../../../v0/util - name: defaultRequestConfig path: ../../../../v0/util + - name: getIntegrationsObj + path: ../../../../v0/util + - name: getFieldValueFromMessage + path: ../../../../v0/util - path: ./utils - path: ./config - path: lodash @@ -55,23 +61,24 @@ steps: ref: https://dev.emarsys.com/docs/core-api-reference/fl0xx6rwfbwqb-trigger-an-external-event condition: $.outputs.messageType === {{$.EventType.TRACK}} template: | - const integrationObject = $.getIntegrationsObj(^.message, 'emersys'); - const emersysIdentifierId = integrationObject?.customIdentifierId || ^.destination.Config.emersysCustomIdentifier || $.EMAIL_FIELD_ID; + const properties = ^.message.properties; + const integrationObject = $.getIntegrationsObj(^.message, 'emarsys'); + const emersysIdentifierId = integrationObject.customIdentifierId ?? ^.destination.Config.emersysCustomIdentifier ?? $.EMAIL_FIELD_ID; const payload = { - key_id: emersysIdentifierId , - external_id: $.deduceExternalIdValue(.message,emersysIdentifierId,.destination.Config.fieldMapping), - trigger_id: integrationObject.trigger_id, + key_id: emersysIdentifierId, + external_id: $.deduceExternalIdValue(^.message,emersysIdentifierId,.destination.Config.fieldMapping), + trigger_id: $.integrationObject.trigger_id, data: properties.data, attachment: Array.isArray(properties.attachment) ? properties.attachment : [properties.attachment], - event_time: $.getFieldValueFromMessage(message, 'timestamp'), + event_time: $.getFieldValueFromMessage(^.message, 'timestamp'), }; $.context.payload = { eventType: ^.message.type, destinationPayload: { payload: $.removeUndefinedAndNullValues(payload), - eventId: $.deduceEventId(.message,.destination.Config), + eventId: $.deduceEventId(^.message,.destination.Config), }, }; - name: buildResponse diff --git a/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml b/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml index 36df10bb43..0e7132ccad 100644 --- a/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml +++ b/src/cdk/v2/destinations/emarsys/rtWorkflow.yaml @@ -32,7 +32,6 @@ steps: description: Batches the successfulEvents template: | $.context.batchedPayload = $.batchResponseBuilder($.outputs.successfulEvents); - console.log("batchedPayload",JSON.stringify($.context.batchedPayload)); - name: finalPayload template: | diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index 9d7faff01b..314725d49d 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -7,13 +7,13 @@ const { removeUndefinedAndNullAndEmptyValues, removeUndefinedAndNullValues, isDefinedAndNotNull, - getHashFromArray, } = require('@rudderstack/integrations-lib'); const { getIntegrationsObj, validateEventName, getValueFromMessage, getFieldValueFromMessage, + getHashFromArray, } = require('../../../../v0/util'); const { EMAIL_FIELD_ID, @@ -23,7 +23,6 @@ const { MAX_BATCH_SIZE_BYTES, groupedSuccessfulPayload, } = require('./config'); -const { SUPPORTED_EVENT_TYPE } = require('../the_trade_desk_real_time_conversions/config'); const { EventType } = require('../../../../constants'); const base64Sha = (str) => { @@ -115,13 +114,23 @@ const deduceExternalIdValue = (message, emersysIdentifier, fieldMapping) => { emersysIdentifier, fieldMapping, ); - const externalIdValue = getValueFromMessage(message.context.traits, configuredPayloadProperty); + const externalIdValue = getValueFromMessage(message, [ + `traits.${configuredPayloadProperty}`, + `context.traits.${configuredPayloadProperty}`, + ]); + + if (!isDefinedAndNotNull(deduceExternalIdValue)) { + throw new InstrumentationError( + `Could not find value for externalId required in ${message.type} call. Aborting.`, + ); + } + return externalIdValue; }; -const buildGroupPayload = (message, destination) => { - const { emersysCustomIdentifier, defaultContactList, fieldMapping } = destination.Config; - const integrationObject = getIntegrationsObj(message, 'emersys'); +const buildGroupPayload = (message, destConfig) => { + const { emersysCustomIdentifier, defaultContactList, fieldMapping } = destConfig; + const integrationObject = getIntegrationsObj(message, 'emarsys'); const emersysIdentifier = integrationObject?.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; const externalIdValue = deduceExternalIdValue(message, emersysIdentifier, fieldMapping); @@ -155,6 +164,7 @@ const deduceEventId = (message, destConfig) => { if (!eventId) { throw new ConfigurationError(`${event} is not mapped to any Emersys external event. Aborting`); } + return eventId; }; const buildTrackPayload = (message, destination) => { @@ -197,7 +207,7 @@ const deduceEndPoint = (finalPayload) => { case EventType.IDENTIFY: endPoint = 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1'; break; - case SUPPORTED_EVENT_TYPE.GROUP: + case EventType.GROUP: contactListId = destinationPayload.contactListId; endPoint = `https://api.emarsys.net/api/v2/contactlist/${contactListId}/add`; break; @@ -274,15 +284,15 @@ const createGroupBatches = (events) => { const grouped = lodash.groupBy( events, (item) => - `${item.message.body.JSON.destinationPayload.payload.key_id}-${item.message.body.JSON.destinationPayload.contactListId}`, + `${item.message[0].body.JSON.destinationPayload.payload.key_id}-${item.message[0].body.JSON.destinationPayload.contactListId}`, ); // eslint-disable-next-line @typescript-eslint/no-unused-vars return Object.entries(grouped).flatMap(([key, group]) => { - const keyId = group[0].message.body.JSON.destinationPayload.payload.key_id; - const { contactListId } = group[0].message.body.JSON.destinationPayload; + const keyId = group[0].message[0].body.JSON.destinationPayload.payload.key_id; + const { contactListId } = group[0].message[0].body.JSON.destinationPayload; const combinedExternalIds = group.reduce((acc, item) => { - acc.push(...item.message.body.JSON.destinationPayload.payload.external_ids); + acc.push(...item.message[0].body.JSON.destinationPayload.payload.external_ids); return acc; }, []); @@ -299,11 +309,13 @@ const createGroupBatches = (events) => { }); }; -const createTrackBatches = (events) => ({ - endpoint: events[0].message.endPoint, - payload: events[0].message.body.JSON.destinationPayload, - metadata: events[0].metadata, -}); +const createTrackBatches = (events) => [ + { + endpoint: events[0].message[0].endpoint, + payload: events[0].message[0].body.JSON.destinationPayload.payload, + metadata: [events[0].metadata], + }, +]; const formatIdentifyPayloadsWithEndpoint = (combinedPayloads, endpointUrl = '') => combinedPayloads.map((singleCombinedPayload) => ({ endpoint: endpointUrl, @@ -334,76 +346,149 @@ const buildBatchedRequest = (batches, method, constants, batchedStatus = true) = destination: constants.destination, })); -const batchResponseBuilder = (successfulEvents) => { - const finaloutput = []; - let batchesOfIdentifyEvents; - if (successfulEvents.length === 0) { - return []; - } - const constants = { +// const batchResponseBuilder = (successfulEvents) => { +// const finaloutput = []; +// let batchesOfIdentifyEvents; +// if (successfulEvents.length === 0) { +// return []; +// } +// const constants = { +// version: successfulEvents[0].message[0].version, +// type: successfulEvents[0].message[0].type, +// headers: successfulEvents[0].message[0].headers, +// destination: successfulEvents[0].destination, +// }; + +// const typedEventGroups = lodash.groupBy( +// successfulEvents, +// (event) => event.message[0].body.JSON.eventType, +// ); +// Object.keys(typedEventGroups).forEach((eachEventGroup) => { +// switch (eachEventGroup) { +// case EventType.IDENTIFY: +// batchesOfIdentifyEvents = createIdentifyBatches(typedEventGroups[eachEventGroup]); +// groupedSuccessfulPayload.identify.batches = formatIdentifyPayloadsWithEndpoint( +// batchesOfIdentifyEvents, +// 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', +// ); +// break; +// case EventType.GROUP: +// groupedSuccessfulPayload.group.batches = createGroupBatches( +// typedEventGroups[eachEventGroup], +// ); +// break; +// case EventType.TRACK: +// groupedSuccessfulPayload.track.batches = createTrackBatches( +// typedEventGroups[eachEventGroup], +// ); +// break; +// default: +// break; +// } +// return groupedSuccessfulPayload; +// }); +// // Process each identify batch +// if (groupedSuccessfulPayload.identify) { +// const identifyBatches = buildBatchedRequest( +// groupedSuccessfulPayload.identify.batches, +// groupedSuccessfulPayload.identify.method, +// constants, +// ); +// finaloutput.push(...identifyBatches); +// } + +// // Process each group batch +// if (groupedSuccessfulPayload.group) { +// const groupBatches = buildBatchedRequest( +// groupedSuccessfulPayload.group.batches, +// groupedSuccessfulPayload.group.method, +// constants, +// ); +// finaloutput.push(...groupBatches); +// } + +// if (groupedSuccessfulPayload.track) { +// const trackBatches = buildBatchedRequest( +// groupedSuccessfulPayload.track.batches, +// groupedSuccessfulPayload.track.method, +// constants, +// false, +// ); +// finaloutput.push(...trackBatches); +// } +// return finaloutput; +// }; + +// Helper to initialize the constants used across batch processing +function initializeConstants(successfulEvents) { + if (successfulEvents.length === 0) return null; + return { version: successfulEvents[0].message[0].version, type: successfulEvents[0].message[0].type, headers: successfulEvents[0].message[0].headers, destination: successfulEvents[0].destination, }; +} + +// Helper to append requests based on batched events and constants +function appendRequestsToOutput(groupPayload, output, constants, batched = true) { + if (groupPayload.batches) { + const requests = buildBatchedRequest( + groupPayload.batches, + groupPayload.method, + constants, + batched, + ); + output.push(...requests); + } +} - const typedEventGroups = lodash.groupBy( - successfulEvents, - (event) => event.message[0].body.JSON.eventType, - ); - Object.keys(typedEventGroups).forEach((eachEventGroup) => { - switch (eachEventGroup) { +// Process batches based on event types +function processEventBatches(typedEventGroups, constants) { + let batchesOfIdentifyEvents; + const finalOutput = []; + + // Process each event group based on type + Object.keys(typedEventGroups).forEach((eventType) => { + switch (eventType) { case EventType.IDENTIFY: - batchesOfIdentifyEvents = createIdentifyBatches(typedEventGroups[eachEventGroup]); + batchesOfIdentifyEvents = createIdentifyBatches(typedEventGroups[eventType]); groupedSuccessfulPayload.identify.batches = formatIdentifyPayloadsWithEndpoint( batchesOfIdentifyEvents, 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', ); break; case EventType.GROUP: - groupedSuccessfulPayload.group.batches = createGroupBatches(eachEventGroup); + groupedSuccessfulPayload.group.batches = createGroupBatches(typedEventGroups[eventType]); break; case EventType.TRACK: - groupedSuccessfulPayload.track.batches = createTrackBatches(eachEventGroup); + groupedSuccessfulPayload.track.batches = createTrackBatches(typedEventGroups[eventType]); break; default: break; } - return groupedSuccessfulPayload; }); - console.log('groupedSuccessfulPayload', JSON.stringify(groupedSuccessfulPayload)); - // Process each identify batch - if (groupedSuccessfulPayload.identify) { - const identifyBatches = buildBatchedRequest( - groupedSuccessfulPayload.identify.batches, - groupedSuccessfulPayload.identify.method, - constants, - ); - finaloutput.push(...identifyBatches); - } - // Process each group batch - if (groupedSuccessfulPayload.group) { - const groupBatches = buildBatchedRequest( - groupedSuccessfulPayload.group.batches, - groupedSuccessfulPayload.group.method, - constants, - ); - finaloutput.push(...groupBatches); - } + // Convert batches into requests for each event type and push to final output + appendRequestsToOutput(groupedSuccessfulPayload.identify, finalOutput, constants); + appendRequestsToOutput(groupedSuccessfulPayload.group, finalOutput, constants); + appendRequestsToOutput(groupedSuccessfulPayload.track, finalOutput, constants, false); - if (groupedSuccessfulPayload.track) { - const trackBatches = buildBatchedRequest( - groupedSuccessfulPayload.track.batches, - groupedSuccessfulPayload.track.method, - constants, - false, - ); - finaloutput.push(...trackBatches); - } - console.log('FINAL', JSON.stringify(finaloutput)); - return finaloutput; -}; + return finalOutput; +} + +// Entry function to create batches from successful events +function batchResponseBuilder(successfulEvents) { + const constants = initializeConstants(successfulEvents); + if (!constants) return []; + + const typedEventGroups = lodash.groupBy( + successfulEvents, + (event) => event.message[0].body.JSON.eventType, + ); + + return processEventBatches(typedEventGroups, constants); +} module.exports = { buildIdentifyPayload, diff --git a/test/integrations/destinations/emarsys/router/data.ts b/test/integrations/destinations/emarsys/router/data.ts index 32f5782474..c0b5216781 100644 --- a/test/integrations/destinations/emarsys/router/data.ts +++ b/test/integrations/destinations/emarsys/router/data.ts @@ -9,6 +9,10 @@ const config = { from: 'Order Completed', to: 'purchase', }, + { + from: 'Product Added', + to: 'addToCart', + }, ], fieldMapping: [ { @@ -90,9 +94,6 @@ export const data = [ destinationType: '', namespace: '', jobId: 1, - secret: { - accessToken: 'dummyToken', - }, }, destination: commonDestination, }, @@ -123,9 +124,6 @@ export const data = [ destinationType: '', namespace: '', jobId: 2, - secret: { - accessToken: 'dummyToken', - }, }, destination: commonDestination, }, @@ -160,9 +158,6 @@ export const data = [ destinationType: '', namespace: '', jobId: 3, - secret: { - accessToken: 'dummyToken', - }, }, destination: commonDestination, }, @@ -197,9 +192,164 @@ export const data = [ destinationType: '', namespace: '', jobId: 4, - secret: { - accessToken: 'dummyToken', + }, + destination: commonDestination, + }, + { + message: { + anonymousId: 'anonId06', + channel: 'web', + context: { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', + traits: { + email: 'abc@gmail.com', + lastName: 'Doe', + firstName: 'John', + }, }, + integrations: { + All: true, + }, + traits: { + company: 'testComp', + }, + originalTimestamp: '2020-01-24T06:29:02.362Z', + receivedAt: '2020-01-24T11:59:02.403+05:30', + request_ip: '[::1]:53709', + sentAt: '2020-01-24T06:29:02.363Z', + timestamp: '2023-07-06T11:59:02.402+05:30', + type: 'group', + userId: 'userId06', + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 5, + }, + destination: commonDestination, + }, + { + message: { + anonymousId: 'anonId06', + channel: 'web', + context: { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', + traits: { + email: 'abc2@gmail.com', + lastName: 'Doe2', + firstName: 'John2', + }, + }, + integrations: { + All: true, + }, + traits: { + company: 'testComp', + }, + originalTimestamp: '2020-01-24T06:29:02.362Z', + receivedAt: '2020-01-24T11:59:02.403+05:30', + request_ip: '[::1]:53709', + sentAt: '2020-01-24T06:29:02.363Z', + timestamp: '2023-07-06T11:59:02.402+05:30', + type: 'group', + userId: 'userId06', + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 6, + }, + destination: commonDestination, + }, + { + message: { + channel: 'web', + context: { + traits: { + firstName: 'test', + lastName: 'one', + }, + }, + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + event: 'product list viewed', + userId: 'testuserId1', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 7, + }, + destination: commonDestination, + }, + { + message: { + event: 'Order Completed', + anonymousId: 'anonId06', + channel: 'web', + context: { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', + traits: { + email: 'abc@gmail.com', + lastName: 'Doe', + firstName: 'John', + }, + }, + integrations: { + All: true, + }, + properties: { + company: 'testComp', + data: { + section_group1: [ + { + section_variable1: 'some_value', + section_variable2: 'another_value', + }, + { + section_variable1: 'yet_another_value', + section_variable2: 'one_more_value', + }, + ], + global: { + global_variable1: 'global_value', + global_variable2: 'another_global_value', + }, + }, + attachment: [ + { + filename: 'example.pdf', + data: 'ZXhhbXBsZQo=', + }, + ], + }, + messageId: '2536eda4-d638-4c93-8014-8ffe3f083214', + originalTimestamp: '2020-01-24T06:29:02.362Z', + receivedAt: '2020-01-24T11:59:02.403+05:30', + request_ip: '[::1]:53709', + sentAt: '2020-01-24T06:29:02.363Z', + timestamp: '2023-07-06T11:59:02.402+05:30', + type: 'track', + userId: 'userId06', + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 8, }, destination: commonDestination, }, @@ -217,38 +367,41 @@ export const data = [ { metadata: [ { - destinationType: '', - jobId: 1, - namespace: '', - secret: { - accessToken: 'dummyToken', - }, sourceType: '', - }, - { destinationType: '', - jobId: 2, namespace: '', - secret: { - accessToken: 'dummyToken', - }, - sourceType: '', + jobId: 7, }, ], + batched: false, + statusCode: 400, + error: + 'Either configured custom contact identifier value or default identifier email value is missing', + statTags: { + destType: 'EMARSYS', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'cdkV2', + module: 'destination', + }, + destination: commonDestination, + }, + { batchedRequest: { body: { JSON: { key_id: '3', contacts: [ { - '3': 'testone@gmail.com', '1': 'test', '2': 'one', + '3': 'testone@gmail.com', }, { - '3': 'testtwo@gmail.com', '1': 'test', '2': 'one', + '3': 'testtwo@gmail.com', }, ], contact_list_id: 'dummy', @@ -265,36 +418,82 @@ export const data = [ 'Content-Type': 'application/json', Accept: 'application/json', 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="OWM2ODlmYjZiMDA0YTQwZjc1NjkyOWFiZTA1MTQ0ZmUwOGYyYWQ2NA==", Nonce="8c02af01eb527f450340bb82ebd40dde", Created="2024-05-02T15:41:20.529Z"', + 'UsernameToken Username="dummy", PasswordDigest="MGI5ZTdkYzgzMTA2Y2E0NzNlOTc1ZDEyY2I0NThhOGMxOTdjZGJlOQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2024-05-03T10:53:59.547Z"', }, params: {}, files: {}, }, + metadata: [ + { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 1, + }, + { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 2, + }, + ], batched: true, statusCode: 200, destination: commonDestination, }, { + batchedRequest: { + body: { + JSON: { + key_id: '1', + contacts: [ + { + '1': 'test', + '2': 'one', + '3': 'testtwo@gmail.com', + }, + ], + contact_list_id: 'dummy2', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'PUT', + endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="MGI5ZTdkYzgzMTA2Y2E0NzNlOTc1ZDEyY2I0NThhOGMxOTdjZGJlOQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2024-05-03T10:53:59.547Z"', + }, + params: {}, + files: {}, + }, metadata: [ { + sourceType: '', destinationType: '', - jobId: 3, namespace: '', - secret: { - accessToken: 'dummyToken', - }, - sourceType: '', + jobId: 3, }, ], + batched: true, + statusCode: 200, + destination: commonDestination, + }, + { batchedRequest: { body: { JSON: { - key_id: '1', + key_id: '2', contacts: [ { - '3': 'testtwo@gmail.com', '1': 'test', '2': 'one', + '3': 'testtwo@gmail.com', }, ], contact_list_id: 'dummy2', @@ -311,39 +510,94 @@ export const data = [ 'Content-Type': 'application/json', Accept: 'application/json', 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="OWM2ODlmYjZiMDA0YTQwZjc1NjkyOWFiZTA1MTQ0ZmUwOGYyYWQ2NA==", Nonce="8c02af01eb527f450340bb82ebd40dde", Created="2024-05-02T15:41:20.529Z"', + 'UsernameToken Username="dummy", PasswordDigest="MGI5ZTdkYzgzMTA2Y2E0NzNlOTc1ZDEyY2I0NThhOGMxOTdjZGJlOQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2024-05-03T10:53:59.547Z"', }, params: {}, files: {}, }, + metadata: [ + { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 4, + }, + ], batched: true, statusCode: 200, destination: commonDestination, }, { + batchedRequest: { + body: { + JSON: { + key_id: '3', + external_ids: ['abc@gmail.com', 'abc2@gmail.com'], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/contactlist/dummy/add', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="MGI5ZTdkYzgzMTA2Y2E0NzNlOTc1ZDEyY2I0NThhOGMxOTdjZGJlOQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2024-05-03T10:53:59.547Z"', + }, + params: {}, + files: {}, + }, metadata: [ { + sourceType: '', destinationType: '', - jobId: 4, namespace: '', - secret: { - accessToken: 'dummyToken', - }, + jobId: 5, + }, + { sourceType: '', + destinationType: '', + namespace: '', + jobId: 6, }, ], + batched: true, + statusCode: 200, + destination: commonDestination, + }, + { batchedRequest: { body: { JSON: { - key_id: '2', - contacts: [ + key_id: '3', + external_id: 'abc@gmail.com', + data: { + section_group1: [ + { + section_variable1: 'some_value', + section_variable2: 'another_value', + }, + { + section_variable1: 'yet_another_value', + section_variable2: 'one_more_value', + }, + ], + global: { + global_variable1: 'global_value', + global_variable2: 'another_global_value', + }, + }, + attachment: [ { - '3': 'testtwo@gmail.com', - '1': 'test', - '2': 'one', + filename: 'example.pdf', + data: 'ZXhhbXBsZQo=', }, ], - contact_list_id: 'dummy2', + event_time: '2023-07-06T11:59:02.402+05:30', }, JSON_ARRAY: {}, XML: {}, @@ -351,18 +605,26 @@ export const data = [ }, version: '1', type: 'REST', - method: 'PUT', - endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/event/purchase/trigger', headers: { 'Content-Type': 'application/json', Accept: 'application/json', 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="OWM2ODlmYjZiMDA0YTQwZjc1NjkyOWFiZTA1MTQ0ZmUwOGYyYWQ2NA==", Nonce="8c02af01eb527f450340bb82ebd40dde", Created="2024-05-02T15:41:20.529Z"', + 'UsernameToken Username="dummy", PasswordDigest="MGI5ZTdkYzgzMTA2Y2E0NzNlOTc1ZDEyY2I0NThhOGMxOTdjZGJlOQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2024-05-03T10:53:59.547Z"', }, params: {}, files: {}, }, - batched: true, + metadata: [ + { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 8, + }, + ], + batched: false, statusCode: 200, destination: commonDestination, }, From 8b05a626bb0ebbb93c7423298d9ed38ca3540c51 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Fri, 3 May 2024 22:55:20 +0530 Subject: [PATCH 13/26] fix: cide clean up --- src/cdk/v2/destinations/emarsys/utils.js | 73 ------------------------ 1 file changed, 73 deletions(-) diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index 314725d49d..8e82846f58 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -346,79 +346,6 @@ const buildBatchedRequest = (batches, method, constants, batchedStatus = true) = destination: constants.destination, })); -// const batchResponseBuilder = (successfulEvents) => { -// const finaloutput = []; -// let batchesOfIdentifyEvents; -// if (successfulEvents.length === 0) { -// return []; -// } -// const constants = { -// version: successfulEvents[0].message[0].version, -// type: successfulEvents[0].message[0].type, -// headers: successfulEvents[0].message[0].headers, -// destination: successfulEvents[0].destination, -// }; - -// const typedEventGroups = lodash.groupBy( -// successfulEvents, -// (event) => event.message[0].body.JSON.eventType, -// ); -// Object.keys(typedEventGroups).forEach((eachEventGroup) => { -// switch (eachEventGroup) { -// case EventType.IDENTIFY: -// batchesOfIdentifyEvents = createIdentifyBatches(typedEventGroups[eachEventGroup]); -// groupedSuccessfulPayload.identify.batches = formatIdentifyPayloadsWithEndpoint( -// batchesOfIdentifyEvents, -// 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', -// ); -// break; -// case EventType.GROUP: -// groupedSuccessfulPayload.group.batches = createGroupBatches( -// typedEventGroups[eachEventGroup], -// ); -// break; -// case EventType.TRACK: -// groupedSuccessfulPayload.track.batches = createTrackBatches( -// typedEventGroups[eachEventGroup], -// ); -// break; -// default: -// break; -// } -// return groupedSuccessfulPayload; -// }); -// // Process each identify batch -// if (groupedSuccessfulPayload.identify) { -// const identifyBatches = buildBatchedRequest( -// groupedSuccessfulPayload.identify.batches, -// groupedSuccessfulPayload.identify.method, -// constants, -// ); -// finaloutput.push(...identifyBatches); -// } - -// // Process each group batch -// if (groupedSuccessfulPayload.group) { -// const groupBatches = buildBatchedRequest( -// groupedSuccessfulPayload.group.batches, -// groupedSuccessfulPayload.group.method, -// constants, -// ); -// finaloutput.push(...groupBatches); -// } - -// if (groupedSuccessfulPayload.track) { -// const trackBatches = buildBatchedRequest( -// groupedSuccessfulPayload.track.batches, -// groupedSuccessfulPayload.track.method, -// constants, -// false, -// ); -// finaloutput.push(...trackBatches); -// } -// return finaloutput; -// }; - // Helper to initialize the constants used across batch processing function initializeConstants(successfulEvents) { if (successfulEvents.length === 0) return null; From acf6eeb3a91903abb5afcc6f45a2536f194c1b4b Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Sat, 4 May 2024 12:34:32 +0530 Subject: [PATCH 14/26] fix: router test cases finalised --- .../destinations/emarsys/router/data.ts | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/test/integrations/destinations/emarsys/router/data.ts b/test/integrations/destinations/emarsys/router/data.ts index c0b5216781..8f449bd351 100644 --- a/test/integrations/destinations/emarsys/router/data.ts +++ b/test/integrations/destinations/emarsys/router/data.ts @@ -1,3 +1,4 @@ +import crypto from 'crypto'; const config = { discardEmptyProperties: true, emersysUsername: 'dummy', @@ -52,14 +53,23 @@ const commonDestination = { Enabled: true, }; +export const mockFns = (_) => { + // @ts-ignore + jest.useFakeTimers().setSystemTime(new Date('2019-10-14')); + jest.mock('crypto', () => ({ + ...jest.requireActual('crypto'), + randomBytes: jest.fn().mockReturnValue(Buffer.from('5398e214ae99c2e50afb709a3bc423f9', 'hex')), + })); +}; + export const data = [ { id: 'emarsys-track-test-1', name: 'emarsys', - description: 'Track call : custom event calls with simple user properties and traits', + description: 'combined batch', scenario: 'Business', successCriteria: - 'event not respecting the internal mapping and as well as UI mapping should be considered as a custom event and should be sent as it is', + 'Identify, group events should be batched based on audience list and key_id criteria and track should not be batched ', feature: 'router', module: 'destination', version: 'v0', @@ -418,7 +428,7 @@ export const data = [ 'Content-Type': 'application/json', Accept: 'application/json', 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="MGI5ZTdkYzgzMTA2Y2E0NzNlOTc1ZDEyY2I0NThhOGMxOTdjZGJlOQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2024-05-03T10:53:59.547Z"', + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', }, params: {}, files: {}, @@ -467,7 +477,7 @@ export const data = [ 'Content-Type': 'application/json', Accept: 'application/json', 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="MGI5ZTdkYzgzMTA2Y2E0NzNlOTc1ZDEyY2I0NThhOGMxOTdjZGJlOQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2024-05-03T10:53:59.547Z"', + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', }, params: {}, files: {}, @@ -510,7 +520,7 @@ export const data = [ 'Content-Type': 'application/json', Accept: 'application/json', 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="MGI5ZTdkYzgzMTA2Y2E0NzNlOTc1ZDEyY2I0NThhOGMxOTdjZGJlOQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2024-05-03T10:53:59.547Z"', + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', }, params: {}, files: {}, @@ -546,7 +556,7 @@ export const data = [ 'Content-Type': 'application/json', Accept: 'application/json', 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="MGI5ZTdkYzgzMTA2Y2E0NzNlOTc1ZDEyY2I0NThhOGMxOTdjZGJlOQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2024-05-03T10:53:59.547Z"', + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', }, params: {}, files: {}, @@ -611,7 +621,7 @@ export const data = [ 'Content-Type': 'application/json', Accept: 'application/json', 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="MGI5ZTdkYzgzMTA2Y2E0NzNlOTc1ZDEyY2I0NThhOGMxOTdjZGJlOQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2024-05-03T10:53:59.547Z"', + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', }, params: {}, files: {}, @@ -633,4 +643,4 @@ export const data = [ }, }, }, -]; +].map((d) => ({ ...d, mockFns })); From 110aff23277c9a3ee03e61336ce86607f3319a69 Mon Sep 17 00:00:00 2001 From: Shrouti Gangopadhyay Date: Sun, 5 May 2024 18:37:32 +0530 Subject: [PATCH 15/26] small edit --- src/cdk/v2/destinations/emarsys/procWorkflow.yaml | 4 ++-- src/cdk/v2/destinations/emarsys/utils.js | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/cdk/v2/destinations/emarsys/procWorkflow.yaml b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml index ee7f78b743..0d3152767e 100644 --- a/src/cdk/v2/destinations/emarsys/procWorkflow.yaml +++ b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml @@ -63,11 +63,11 @@ steps: template: | const properties = ^.message.properties; const integrationObject = $.getIntegrationsObj(^.message, 'emarsys'); - const emersysIdentifierId = integrationObject.customIdentifierId ?? ^.destination.Config.emersysCustomIdentifier ?? $.EMAIL_FIELD_ID; + const emersysIdentifierId = $.deduceCustomIdentifier(integrationObject, ^.destination.Config.emersysCustomIdentifier); const payload = { key_id: emersysIdentifierId, external_id: $.deduceExternalIdValue(^.message,emersysIdentifierId,.destination.Config.fieldMapping), - trigger_id: $.integrationObject.trigger_id, + trigger_id: integrationObject.trigger_id, data: properties.data, attachment: Array.isArray(properties.attachment) ? properties.attachment diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index 8e82846f58..d443cc0f10 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -47,6 +47,9 @@ const buildHeader = (destConfig) => { }; }; +const deduceCustomIdentifier = (integrationObject, emersysCustomIdentifier) => + integrationObject?.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; + const buildIdentifyPayload = (message, destConfig) => { let destinationPayload; const { fieldMapping, emersysCustomIdentifier, discardEmptyProperties, defaultContactList } = @@ -73,8 +76,7 @@ const buildIdentifyPayload = (message, destConfig) => { } }); } - const emersysIdentifier = - integrationObject?.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; + const emersysIdentifier = deduceCustomIdentifier(integrationObject, emersysCustomIdentifier); const finalPayload = discardEmptyProperties === true ? removeUndefinedAndNullAndEmptyValues(payload) // empty property value has a significance in emersys @@ -131,8 +133,7 @@ const deduceExternalIdValue = (message, emersysIdentifier, fieldMapping) => { const buildGroupPayload = (message, destConfig) => { const { emersysCustomIdentifier, defaultContactList, fieldMapping } = destConfig; const integrationObject = getIntegrationsObj(message, 'emarsys'); - const emersysIdentifier = - integrationObject?.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; + const emersysIdentifier = deduceCustomIdentifier(integrationObject, emersysCustomIdentifier); const externalIdValue = deduceExternalIdValue(message, emersysIdentifier, fieldMapping); if (!isDefinedAndNotNull(externalIdValue)) { throw new InstrumentationError( @@ -434,4 +435,5 @@ module.exports = { buildTrackPayload, deduceExternalIdValue, deduceEventId, + deduceCustomIdentifier, }; From 00b5f4fdd74a3fa5b05c8f10b73e4420c093ea6e Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 6 May 2024 00:49:22 +0530 Subject: [PATCH 16/26] fix: processor test cases finalised --- src/cdk/v2/destinations/emarsys/utils.js | 11 +- .../destinations/emarsys/processor/data.ts | 1417 +++++++++++++++++ 2 files changed, 1423 insertions(+), 5 deletions(-) create mode 100644 test/integrations/destinations/emarsys/processor/data.ts diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index d443cc0f10..dabf49c971 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -57,9 +57,10 @@ const buildIdentifyPayload = (message, destConfig) => { const payload = {}; const integrationObject = getIntegrationsObj(message, 'emarsys'); + console.log('integrationObject', integrationObject); const finalContactList = integrationObject?.contactListId || defaultContactList; - - if (!isDefinedAndNotNullAndNotEmpty(finalContactList)) { + console.log('finalContactList', finalContactList); + if (!isDefinedAndNotNullAndNotEmpty(String(finalContactList))) { throw new InstrumentationError( 'Cannot a find a specific contact list either through configuration or via integrations object', ); @@ -105,10 +106,10 @@ const buildIdentifyPayload = (message, destConfig) => { }; const findRudderPropertyByEmersysProperty = (emersysProperty, fieldMapping) => { - // Use lodash to find the object where the emersysProperty matches the input - const item = lodash.find(fieldMapping, { emersysProperty }); + // find the object where the emersysProperty matches the input + const item = lodash.find(fieldMapping, { emersysProperty: String(emersysProperty) }); // Return the rudderProperty if the object is found, otherwise return null - return item ? item.rudderProperty : null; + return item ? item.rudderProperty : 'email'; }; const deduceExternalIdValue = (message, emersysIdentifier, fieldMapping) => { diff --git a/test/integrations/destinations/emarsys/processor/data.ts b/test/integrations/destinations/emarsys/processor/data.ts new file mode 100644 index 0000000000..f3181f3d81 --- /dev/null +++ b/test/integrations/destinations/emarsys/processor/data.ts @@ -0,0 +1,1417 @@ +export const mockFns = (_) => { + // @ts-ignore + jest.useFakeTimers().setSystemTime(new Date('2019-10-14')); + jest.mock('crypto', () => ({ + ...jest.requireActual('crypto'), + randomBytes: jest.fn().mockReturnValue(Buffer.from('5398e214ae99c2e50afb709a3bc423f9', 'hex')), + })); +}; + +export const data = [ + { + name: 'emarsys', + description: 'Test 1 : Track call custom identifier mapped from destination config ', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + metadata: {}, + message: { + event: 'Order Completed', + anonymousId: 'anonId06', + channel: 'web', + context: { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', + traits: { + email: 'abc@gmail.com', + lastName: 'Doe', + firstName: 'John', + }, + }, + integrations: { + All: true, + EMARSYS: { + trigger_id: 'EVENT_TRIGGER_ID', + }, + }, + properties: { + company: 'testComp', + data: { + section_group1: [ + { + section_variable1: 'some_value', + section_variable2: 'another_value', + }, + { + section_variable1: 'yet_another_value', + section_variable2: 'one_more_value', + }, + ], + global: { + global_variable1: 'global_value', + global_variable2: 'another_global_value', + }, + }, + attachment: [ + { + filename: 'example.pdf', + data: 'ZXhhbXBsZQo=', + }, + ], + }, + messageId: '2536eda4-d638-4c93-8014-8ffe3f083214', + originalTimestamp: '2020-01-24T06:29:02.362Z', + receivedAt: '2020-01-24T11:59:02.403+05:30', + request_ip: '[::1]:53709', + sentAt: '2020-01-24T06:29:02.363Z', + timestamp: '2023-07-06T11:59:02.402+05:30', + type: 'track', + userId: 'userId06', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '3', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/event/purchase/trigger', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', + }, + params: {}, + body: { + JSON: { + eventType: 'track', + destinationPayload: { + payload: { + key_id: '3', + external_id: 'abc@gmail.com', + trigger_id: 'EVENT_TRIGGER_ID', + data: { + section_group1: [ + { + section_variable1: 'some_value', + section_variable2: 'another_value', + }, + { + section_variable1: 'yet_another_value', + section_variable2: 'one_more_value', + }, + ], + global: { + global_variable1: 'global_value', + global_variable2: 'another_global_value', + }, + }, + attachment: [ + { + filename: 'example.pdf', + data: 'ZXhhbXBsZQo=', + }, + ], + event_time: '2023-07-06T11:59:02.402+05:30', + }, + eventId: 'purchase', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: {}, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: + 'Test 2 : Track call custom identifier mapped from destination config with custom field', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + metadata: {}, + message: { + event: 'Order Completed', + anonymousId: 'anonId06', + channel: 'web', + context: { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', + traits: { + email: 'abc@gmail.com', + lastName: 'Doe', + firstName: 'John', + custom_field: 'value', + }, + }, + integrations: { + All: true, + EMARSYS: { + trigger_id: 'EVENT_TRIGGER_ID', + customIdentifierId: 'custom_id', + }, + }, + properties: { + company: 'testComp', + data: { + section_group1: [ + { + section_variable1: 'some_value', + section_variable2: 'another_value', + }, + { + section_variable1: 'yet_another_value', + section_variable2: 'one_more_value', + }, + ], + global: { + global_variable1: 'global_value', + global_variable2: 'another_global_value', + }, + }, + attachment: [ + { + filename: 'example.pdf', + data: 'ZXhhbXBsZQo=', + }, + ], + }, + messageId: '2536eda4-d638-4c93-8014-8ffe3f083214', + originalTimestamp: '2020-01-24T06:29:02.362Z', + receivedAt: '2020-01-24T11:59:02.403+05:30', + request_ip: '[::1]:53709', + sentAt: '2020-01-24T06:29:02.363Z', + timestamp: '2023-07-06T11:59:02.402+05:30', + type: 'track', + userId: 'userId06', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '3', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'custom_field', + emersysProperty: 'custom_id', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/event/purchase/trigger', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', + }, + params: {}, + body: { + JSON: { + eventType: 'track', + destinationPayload: { + payload: { + key_id: 'custom_id', + external_id: 'value', + trigger_id: 'EVENT_TRIGGER_ID', + data: { + section_group1: [ + { + section_variable1: 'some_value', + section_variable2: 'another_value', + }, + { + section_variable1: 'yet_another_value', + section_variable2: 'one_more_value', + }, + ], + global: { + global_variable1: 'global_value', + global_variable2: 'another_global_value', + }, + }, + attachment: [ + { + filename: 'example.pdf', + data: 'ZXhhbXBsZQo=', + }, + ], + event_time: '2023-07-06T11:59:02.402+05:30', + }, + eventId: 'purchase', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: {}, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'Test 3: Track call with trigger id mapped from integrations object', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + metadata: {}, + message: { + event: 'Order Completed', + anonymousId: 'anonId06', + channel: 'web', + context: { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', + traits: { + email: 'abc@gmail.com', + lastName: 'Doe', + firstName: 'John', + custom_field: 'value', + }, + }, + integrations: { + All: true, + EMARSYS: { + trigger_id: 'EVENT_TRIGGER_ID', + }, + }, + properties: { + company: 'testComp', + data: { + section_group1: [ + { + section_variable1: 'some_value', + section_variable2: 'another_value', + }, + { + section_variable1: 'yet_another_value', + section_variable2: 'one_more_value', + }, + ], + global: { + global_variable1: 'global_value', + global_variable2: 'another_global_value', + }, + }, + attachment: [ + { + filename: 'example.pdf', + data: 'ZXhhbXBsZQo=', + }, + ], + }, + messageId: '2536eda4-d638-4c93-8014-8ffe3f083214', + originalTimestamp: '2020-01-24T06:29:02.362Z', + receivedAt: '2020-01-24T11:59:02.403+05:30', + request_ip: '[::1]:53709', + sentAt: '2020-01-24T06:29:02.363Z', + timestamp: '2023-07-06T11:59:02.402+05:30', + type: 'track', + userId: 'userId06', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'custom_field', + emersysProperty: 'custom_id', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/event/purchase/trigger', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', + }, + params: {}, + body: { + JSON: { + eventType: 'track', + destinationPayload: { + payload: { + key_id: 3, + external_id: 'abc@gmail.com', + trigger_id: 'EVENT_TRIGGER_ID', + data: { + section_group1: [ + { + section_variable1: 'some_value', + section_variable2: 'another_value', + }, + { + section_variable1: 'yet_another_value', + section_variable2: 'one_more_value', + }, + ], + global: { + global_variable1: 'global_value', + global_variable2: 'another_global_value', + }, + }, + attachment: [ + { + filename: 'example.pdf', + data: 'ZXhhbXBsZQo=', + }, + ], + event_time: '2023-07-06T11:59:02.402+05:30', + }, + eventId: 'purchase', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: {}, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'Test 4 : group call with default external id email ', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + metadata: {}, + message: { + anonymousId: 'anonId06', + channel: 'web', + context: { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', + traits: { + email: 'abc@gmail.com', + lastName: 'Doe', + firstName: 'John', + }, + }, + integrations: { + All: true, + }, + traits: { + company: 'testComp', + }, + originalTimestamp: '2020-01-24T06:29:02.362Z', + receivedAt: '2020-01-24T11:59:02.403+05:30', + request_ip: '[::1]:53709', + sentAt: '2020-01-24T06:29:02.363Z', + timestamp: '2023-07-06T11:59:02.402+05:30', + type: 'group', + userId: 'userId06', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'custom_field', + emersysProperty: 'custom_id', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/contactlist/dummy/add', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', + }, + params: {}, + body: { + JSON: { + eventType: 'group', + destinationPayload: { + payload: { + key_id: 3, + external_ids: ['abc@gmail.com'], + }, + contactListId: 'dummy', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: {}, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'Test 5 : group call, custom identifier id mapped from integration object', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + metadata: {}, + message: { + anonymousId: 'anonId06', + channel: 'web', + context: { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', + traits: { + email: 'abc@gmail.com', + lastName: 'Doe', + firstName: 'John', + custom_field: 'value', + }, + }, + integrations: { + All: true, + EMARSYS: { + customIdentifierId: 'custom_id', + }, + }, + traits: { + company: 'testComp', + }, + originalTimestamp: '2020-01-24T06:29:02.362Z', + receivedAt: '2020-01-24T11:59:02.403+05:30', + request_ip: '[::1]:53709', + sentAt: '2020-01-24T06:29:02.363Z', + timestamp: '2023-07-06T11:59:02.402+05:30', + type: 'group', + userId: 'userId06', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'custom_field', + emersysProperty: 'custom_id', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/contactlist/dummy/add', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', + }, + params: {}, + body: { + JSON: { + eventType: 'group', + destinationPayload: { + payload: { + key_id: 'custom_id', + external_ids: ['value'], + }, + contactListId: 'dummy', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: {}, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'Test 6 : custom identifier mapped from destination config', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + metadata: {}, + message: { + anonymousId: 'anonId06', + channel: 'web', + context: { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36', + traits: { + email: 'abc@gmail.com', + lastName: 'Doe', + firstName: 'John', + custom_field: 'value', + }, + }, + integrations: { + All: true, + }, + traits: { + company: 'testComp', + }, + originalTimestamp: '2020-01-24T06:29:02.362Z', + receivedAt: '2020-01-24T11:59:02.403+05:30', + request_ip: '[::1]:53709', + sentAt: '2020-01-24T06:29:02.363Z', + timestamp: '2023-07-06T11:59:02.402+05:30', + type: 'group', + userId: 'userId06', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '2', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'lastName', + emersysProperty: '2', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/contactlist/dummy/add', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', + }, + params: {}, + body: { + JSON: { + eventType: 'group', + destinationPayload: { + payload: { + key_id: '2', + external_ids: ['Doe'], + }, + contactListId: 'dummy', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: {}, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'Test 7 : Identify call with contact list id mapped from integrations objects', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + metadata: {}, + message: { + channel: 'web', + context: { + traits: { + email: 'testone@gmail.com', + firstName: 'test', + lastName: 'one', + 'custom-field': 'value', + }, + }, + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + event: 'product list viewed', + userId: 'testuserId1', + integrations: { + All: true, + EMARSYS: { + contactListId: 123, + }, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '2', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'lastName', + emersysProperty: '2', + }, + { + rudderProperty: 'custom-field', + emersysProperty: 'custom_id', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', + }, + params: {}, + body: { + JSON: { + eventType: 'identify', + destinationPayload: { + key_id: '2', + contacts: [ + { + '2': 'one', + '3': 'testone@gmail.com', + custom_id: 'value', + }, + ], + contact_list_id: 123, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: {}, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'Test 8 : identify call customIdentifierId mapped from integration object', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + metadata: {}, + message: { + channel: 'web', + context: { + traits: { + email: 'testone@gmail.com', + firstName: 'test', + lastName: 'one', + 'custom-field': 'value', + }, + }, + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + userId: 'testuserId1', + integrations: { + All: true, + EMARSYS: { + customIdentifierId: '1', + }, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '2', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'lastName', + emersysProperty: '2', + }, + { + rudderProperty: 'firstName', + emersysProperty: '1', + }, + { + rudderProperty: 'custom-field', + emersysProperty: 'custom_id', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', + }, + params: {}, + body: { + JSON: { + eventType: 'identify', + destinationPayload: { + key_id: '1', + contacts: [ + { + '1': 'test', + '2': 'one', + '3': 'testone@gmail.com', + custom_id: 'value', + }, + ], + contact_list_id: 'dummy', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: {}, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'Test 9 : custom identifier mapped from default email value', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + metadata: {}, + message: { + channel: 'web', + context: { + traits: { + email: 'testone@gmail.com', + firstName: 'test', + lastName: 'one', + 'custom-field': 'value', + }, + }, + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + userId: 'testuserId1', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'lastName', + emersysProperty: '2', + }, + { + rudderProperty: 'firstName', + emersysProperty: '1', + }, + { + rudderProperty: 'custom-field', + emersysProperty: 'custom_id', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', + }, + params: {}, + body: { + JSON: { + eventType: 'identify', + destinationPayload: { + key_id: 3, + contacts: [ + { + '1': 'test', + '2': 'one', + '3': 'testone@gmail.com', + custom_id: 'value', + }, + ], + contact_list_id: 'dummy', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: {}, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'Test 10 : identify call error for not finding custom identifier', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + metadata: {}, + message: { + channel: 'web', + context: { + traits: { + firstName: 'test', + lastName: 'one', + 'custom-field': 'value', + }, + }, + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + userId: 'testuserId1', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '', + defaultContactList: 'dummy', + eventsMapping: [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, + ], + fieldMapping: [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'lastName', + emersysProperty: '2', + }, + { + rudderProperty: 'firstName', + emersysProperty: '1', + }, + { + rudderProperty: 'custom-field', + emersysProperty: 'custom_id', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Either configured custom contact identifier value or default identifier email value is missing: Workflow: procWorkflow, Step: preparePayloadForIdentify, ChildStep: undefined, OriginalError: Either configured custom contact identifier value or default identifier email value is missing', + metadata: {}, + statTags: { + destType: 'EMARSYS', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + }, +].map((d) => ({ ...d, mockFns })); From 2bf77a8a9a5b9c077a9e079f6e004f590aaaec20 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 6 May 2024 12:43:00 +0530 Subject: [PATCH 17/26] fix: data delivery test cases --- src/features.json | 3 +- src/v1/destinations/emarsys/networkHandler.js | 2 +- .../destinations/emarsys/dataDelivery/data.ts | 254 ++++++++++++++++++ .../destinations/emarsys/network.ts | 181 +++++++++++++ test/integrations/testUtils.ts | 3 + 5 files changed, 441 insertions(+), 2 deletions(-) create mode 100644 test/integrations/destinations/emarsys/dataDelivery/data.ts create mode 100644 test/integrations/destinations/emarsys/network.ts diff --git a/src/features.json b/src/features.json index 6d2cac9340..481025b700 100644 --- a/src/features.json +++ b/src/features.json @@ -70,7 +70,8 @@ "KOALA": true, "LINKEDIN_ADS": true, "BLOOMREACH": true, - "MOVABLE_INK": true + "MOVABLE_INK": true, + "EMARSYS": true }, "regulations": [ "BRAZE", diff --git a/src/v1/destinations/emarsys/networkHandler.js b/src/v1/destinations/emarsys/networkHandler.js index 641c89c0de..28d2feba30 100644 --- a/src/v1/destinations/emarsys/networkHandler.js +++ b/src/v1/destinations/emarsys/networkHandler.js @@ -41,7 +41,7 @@ const responseHandler = (responseParams) => { // even if a single event is unsuccessful, the entire batch will fail, we will filter that event out and retry others if (!isHttpStatusSuccess(status)) { - const errorMessage = response.message || 'unknown error format'; + const errorMessage = response.replyText || 'unknown error format'; responseWithIndividualEvents = rudderJobMetadata.map((metadata) => ({ statusCode: status, metadata, diff --git a/test/integrations/destinations/emarsys/dataDelivery/data.ts b/test/integrations/destinations/emarsys/dataDelivery/data.ts new file mode 100644 index 0000000000..cf27c32865 --- /dev/null +++ b/test/integrations/destinations/emarsys/dataDelivery/data.ts @@ -0,0 +1,254 @@ +import { generateProxyV1Payload } from '../../../testUtils'; +import { ProxyV1TestData } from '../../../testTypes'; + +export const headerBlockWithCorrectAccessToken = { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', +}; + +export const headerBlockWithWrongAccessToken = { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy2", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', +}; + +export const correctContactCreateUpdateData = [ + { + '2': 'Person0', + '3': 'person0@example.com', + '10569': 'efghi', + '10519': 'efghi', + '31': 1, + '39': 'abc', + }, + { + '2': true, + '3': 'abcde', + '10569': 'efgh', + '10519': 1234, + '31': 2, + '39': 'abc', + }, +]; + +export const wrongContactCreateUpdateData = [ + { + '2': 'Person0', + '3': 'person0@example.com', + '10569': 'efghi', + '10519': 'efghi', + '31': 1, + '39': 'abc', + }, + { + '2': true, + '3': 1234, + '10569': 'efgh', + '10519': 1234, + '31': 2, + '39': 'abc', + }, +]; + +export const contactPayload = { + key_id: 10569, + contacts: correctContactCreateUpdateData, + contact_list_id: 'dummy', +}; + +export const correctGroupCallPayload = { + key_id: 'right_id', + external_ids: ['personABC@example.com'], +}; + +export const groupPayloadWithWrongKeyId = { + key_id: 'wrong_id', + external_ids: ['efghi', 'jklmn', 'unknown', 'person4@example.com'], +}; + +export const groupPayloadWithWrongExternalId = { + key_id: 'wrong_id', + external_ids: ['efghi', 'jklmn', 'unknown', 'person4@example.com'], +}; + +export const statTags = { + destType: 'emarsys', + errorCategory: 'network', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', +}; + +export const metadata = { + jobId: 1, + attemptNum: 1, + userId: 'default-userId', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + sourceId: 'default-sourceId', + secret: { + accessToken: 'default-accessToken', + }, + dontBatch: false, +}; + +// const commonIdentifyRequestParametersWithWrongData = { +// headers: headerBlockWithCorrectAccessToken, +// JSON: { ...contactPayload, contacts: wrongContactCreateUpdateData }, +// }; + +const commonIdentifyRequestParameters = { + headers: headerBlockWithCorrectAccessToken, + JSON: { ...contactPayload }, +}; + +const commonIdentifyRequestParametersWithWrongKeyId = { + headers: headerBlockWithCorrectAccessToken, + JSON: { ...contactPayload, key_id: 100 }, +}; + +// const commonGroupRequestParametersWithWrongData = { +// headers: headerBlockWithCorrectAccessToken, +// JSON: groupPayloadWithWrongExternalId, +// }; + +// const commonGroupRequestParameters = { +// headers: headerBlockWithCorrectAccessToken, +// JSON: correctGroupCallPayload, +// }; + +// const commonGroupRequestParametersWithWrongKeyId = { +// headers: headerBlockWithCorrectAccessToken, +// JSON: groupPayloadWithWrongKeyId, +// }; + +export const testScenariosForV1API: ProxyV1TestData[] = [ + // { + // id: 'emarsys_v1_scenario_1', + // name: 'emarsys', + // description: 'Identify Event fails due to wrong key_id', + // successCriteria: 'Should return 400 and aborted', + // scenario: 'Business', + // feature: 'dataDelivery', + // module: 'destination', + // version: 'v1', + // input: { + // request: { + // body: generateProxyV1Payload({ + // endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + // ...commonIdentifyRequestParametersWithWrongKeyId, + // }), + // method: 'PUT', + // }, + // }, + // output: { + // response: { + // status: 200, + // body: { + // output: { + // message: + // "emersys Conversion API: Error transformer proxy v1 during emersys Conversion API response transformation. Incorrect conversions information provided. Conversion's method should be CONVERSIONS_API, indices [0] (0-indexed)", + // response: [ + // { + // error: + // '{"message":"Incorrect conversions information provided. Conversion\'s method should be CONVERSIONS_API, indices [0] (0-indexed)","status":400}', + // statusCode: 400, + // metadata, + // }, + // ], + // statTags, + // status: 400, + // }, + // }, + // }, + // }, + // }, + { + id: 'emarsys_v1_scenario_1', + name: 'emarsys', + description: 'Identify Event fails due to wrong key_id', + successCriteria: 'Should return 400 and aborted', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload({ + endpoint: `https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1`, + ...commonIdentifyRequestParameters, + }), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + "emersys Conversion API: Error transformer proxy v1 during emersys Conversion API response transformation. Incorrect conversions information provided. Conversion's method should be CONVERSIONS_API, indices [0] (0-indexed)", + response: [ + { + error: + '{"message":"Incorrect conversions information provided. Conversion\'s method should be CONVERSIONS_API, indices [0] (0-indexed)","status":400}', + statusCode: 400, + metadata, + }, + ], + statTags, + status: 400, + }, + }, + }, + }, + }, + // { + // id: 'emarsys_v1_scenario_1', + // name: 'emarsys', + // description: 'Identify Event fails due to wrong key_id', + // successCriteria: 'Should return 400 and aborted', + // scenario: 'Business', + // feature: 'dataDelivery', + // module: 'destination', + // version: 'v1', + // input: { + // request: { + // body: generateProxyV1Payload({ + // endpoint: `https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1`, + // ...commonIdentifyRequestParametersWithWrongData, + // }), + // method: 'POST', + // }, + // }, + // output: { + // response: { + // status: 200, + // body: { + // output: { + // message: + // "emersys Conversion API: Error transformer proxy v1 during emersys Conversion API response transformation. Incorrect conversions information provided. Conversion's method should be CONVERSIONS_API, indices [0] (0-indexed)", + // response: [ + // { + // error: + // '{"message":"Incorrect conversions information provided. Conversion\'s method should be CONVERSIONS_API, indices [0] (0-indexed)","status":400}', + // statusCode: 400, + // metadata, + // }, + // ], + // statTags, + // status: 400, + // }, + // }, + // }, + // }, + // }, +]; + +export const data = [...testScenariosForV1API]; diff --git a/test/integrations/destinations/emarsys/network.ts b/test/integrations/destinations/emarsys/network.ts new file mode 100644 index 0000000000..6fab8b8c38 --- /dev/null +++ b/test/integrations/destinations/emarsys/network.ts @@ -0,0 +1,181 @@ +export const headerBlockWithCorrectAccessToken = { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', +}; + +export const headerBlockWithWrongAccessToken = { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-WSSE': + 'UsernameToken Username="dummy2", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', +}; + +export const correctContactCreateUpdateData = [ + { + '2': 'Person0', + '3': 'person0@example.com', + '10569': 'efghi', + '10519': 'efghi', + '31': 1, + '39': 'abc', + }, + { + '2': true, + '3': 'abcde', + '10569': 'efgh', + '10519': 1234, + '31': 2, + '39': 'abc', + }, +]; + +export const wrongContactCreateUpdateData = [ + { + '2': 'Person0', + '3': 'person0@example.com', + '10569': 'efghi', + '10519': 'efghi', + '31': 1, + '39': 'abc', + }, + { + '2': true, + '3': 1234, + '10569': 'efgh', + '10519': 1234, + '31': 2, + '39': 'abc', + }, +]; + +export const contactPayload = { + key_id: 10569, + contacts: correctContactCreateUpdateData, + contact_list_id: 'dummy', +}; + +export const correctGroupCallPayload = { + key_id: 'right_id', + external_ids: ['personABC@example.com'], +}; + +export const groupPayloadWithWrongKeyId = { + key_id: 'wrong_id', + external_ids: ['efghi', 'jklmn', 'unknown', 'person4@example.com'], +}; + +export const groupPayloadWithWrongExternalId = { + key_id: 'wrong_id', + external_ids: ['efghi', 'jklmn', 'unknown', 'person4@example.com'], +}; + +// MOCK DATA +const businessMockData = [ + { + description: 'Mock response from destination depicting request with a correct contact payload', + httpReq: { + method: 'PUT', + url: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + headers: headerBlockWithCorrectAccessToken, + data: contactPayload, + }, + httpRes: { replyCode: 0, replyText: 'OK', data: { ids: ['138621551', 968984932] } }, + }, + { + description: + 'Mock response from destination depicting request with a partially wrong contact payload', + httpReq: { + method: 'PUT', + url: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + headers: headerBlockWithCorrectAccessToken, + data: { ...contactPayload, contacts: wrongContactCreateUpdateData }, + }, + httpRes: { + data: { + ids: ['138621551'], + errors: { '1234': { '2010': 'Contacts with the external id already exist: 3' } }, + }, + status: 200, + }, + }, + { + description: 'Mock response from destination depicting request with a wrong key_id in payload', + httpReq: { + method: 'PUT', + url: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + headers: headerBlockWithCorrectAccessToken, + data: { ...contactPayload, key_id: 100 }, + }, + httpRes: { + data: { ids: [], errors: { '': { '2004': 'Invalid key field id: 100' } } }, + status: 200, + }, + }, + { + description: 'Mock response from destination for correct group call ', + httpReq: { + method: 'put', + url: 'https://api.emarsys.net/api/v2/contactlist/900337462/add', + headers: headerBlockWithCorrectAccessToken, + data: correctGroupCallPayload, + }, + httpRes: { + data: { replyCode: 0, replyText: 'OK', data: { inserted_contacts: 1, errors: [] } }, + status: 200, + }, + }, + { + description: 'Mock response from destination for group call with wrong key_id ', + httpReq: { + method: 'POST', + url: 'https://api.emarsys.net/api/v2/contactlist/900337462/add', + headers: headerBlockWithCorrectAccessToken, + data: groupPayloadWithWrongKeyId, + }, + httpRes: { + data: { replyCode: 2004, replyText: 'Invalid key field id: 100', data: '' }, + status: 400, + }, + }, + { + description: 'Mock response from destination for group call with wrong key_id ', + httpReq: { + method: 'POST', + url: 'https://api.emarsys.net/api/v2/contactlist/900337462/add', + headers: headerBlockWithCorrectAccessToken, + data: groupPayloadWithWrongExternalId, + }, + httpRes: { + data: { + replyCode: 0, + replyText: 'OK', + data: { + inserted_contacts: 0, + errors: { + efghi: { '2008': 'No contact found with the external id: 3' }, + jklmn: { '2008': 'No contact found with the external id: 3' }, + unknown: { '2008': 'No contact found with the external id: 3' }, + }, + }, + }, + status: 200, + }, + }, + { + description: 'Mock response from destination for correct group call, with wrong contact list ', + httpReq: { + method: 'POST', + url: 'https://api.emarsys.net/api/v2/contactlist/wrong-id/add', + headers: headerBlockWithCorrectAccessToken, + data: correctGroupCallPayload, + }, + httpRes: { + data: { replyCode: 1, replyText: 'Action Wrong-id is invalid.', data: '' }, + status: 400, + }, + }, +]; + +export const networkCallsData = [...businessMockData]; diff --git a/test/integrations/testUtils.ts b/test/integrations/testUtils.ts index 0a2727f4d0..eacdce8b39 100644 --- a/test/integrations/testUtils.ts +++ b/test/integrations/testUtils.ts @@ -22,10 +22,13 @@ import { const generateAlphanumericId = (size = 36) => [...Array(size)].map(() => ((Math.random() * size) | 0).toString(size)).join(''); export const getTestDataFilePaths = (dirPath: string, opts: OptionValues): string[] => { + console.log('dirPath', dirPath); const globPattern = join(dirPath, '**', 'data.ts'); let testFilePaths = globSync(globPattern); + // console.log('testFilePaths', JSON.stringify(testFilePaths)); if (opts.destination) { testFilePaths = testFilePaths.filter((testFile) => testFile.includes(opts.destination)); + console.log('testFilePaths', JSON.stringify(testFilePaths)); } if (opts.feature) { testFilePaths = testFilePaths.filter((testFile) => testFile.includes(opts.feature)); From 9fe98aa30eec7712db6d88963f43e7a5cdc7d4a5 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 6 May 2024 16:21:05 +0530 Subject: [PATCH 18/26] fix: adding user deletion implementation --- src/cdk/v2/destinations/emarsys/utils.js | 8 +++- src/v0/destinations/emarsys/deleteUsers.js | 48 ++++++++++--------- src/v0/util/deleteUserUtils.js | 14 +++++- src/v1/destinations/emarsys/networkHandler.js | 1 - 4 files changed, 45 insertions(+), 26 deletions(-) diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index dabf49c971..aa551124ae 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -40,6 +40,12 @@ const getWsseHeader = (user, secret) => { const buildHeader = (destConfig) => { const { emersysUsername, emersysUserSecret } = destConfig; + if ( + !isDefinedAndNotNullAndNotEmpty(emersysUsername) || + !isDefinedAndNotNullAndNotEmpty(emersysUserSecret) + ) { + throw new ConfigurationError('Either Emarsys user name or user secret is missing. Aborting'); + } return { 'Content-Type': 'application/json', Accept: 'application/json', @@ -57,9 +63,7 @@ const buildIdentifyPayload = (message, destConfig) => { const payload = {}; const integrationObject = getIntegrationsObj(message, 'emarsys'); - console.log('integrationObject', integrationObject); const finalContactList = integrationObject?.contactListId || defaultContactList; - console.log('finalContactList', finalContactList); if (!isDefinedAndNotNullAndNotEmpty(String(finalContactList))) { throw new InstrumentationError( 'Cannot a find a specific contact list either through configuration or via integrations object', diff --git a/src/v0/destinations/emarsys/deleteUsers.js b/src/v0/destinations/emarsys/deleteUsers.js index 01044adcd1..42a1bd4d8a 100644 --- a/src/v0/destinations/emarsys/deleteUsers.js +++ b/src/v0/destinations/emarsys/deleteUsers.js @@ -1,4 +1,4 @@ -const { NetworkError, ConfigurationError } = require('@rudderstack/integrations-lib'); +const { NetworkError } = require('@rudderstack/integrations-lib'); const { httpPOST } = require('../../../adapters/network'); const { processAxiosResponse, @@ -7,8 +7,12 @@ const { const { isHttpStatusSuccess } = require('../../util'); const { executeCommonValidations } = require('../../util/regulation-api'); const tags = require('../../util/tags'); -const { getUserIdBatches } = require('../../util/deleteUserUtils'); -const { JSON_MIME_TYPE } = require('../../util/constant'); +const { getCustomIdBatches } = require('../../util/deleteUserUtils'); +const { + buildHeader, + deduceCustomIdentifier, + findRudderPropertyByEmersysProperty, +} = require('../../../cdk/v2/destinations/emarsys/utils'); /** * This function will help to delete the users one by one from the userAttributes array. @@ -17,39 +21,39 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); * @returns */ const userDeletionHandler = async (userAttributes, config) => { - const { apiKey } = config; - - if (!apiKey) { - throw new ConfigurationError('Api Key is required for user deletion'); - } - - const endpoint = 'https://api.sprig.com/v2/purge/visitors'; - const headers = { - Accept: JSON_MIME_TYPE, - 'Content-Type': JSON_MIME_TYPE, - Authorization: `API-Key ${apiKey}`, - }; + const endpoint = 'https://api.emarsys.net/api/v2/contact/delete'; + const headers = buildHeader(config); + const customIdentifier = deduceCustomIdentifier({}, config.emersysCustomIdentifier); + const configuredPayloadProperty = findRudderPropertyByEmersysProperty( + customIdentifier, + config.fieldMapping, + ); /** - * userIdBatches = [[u1,u2,u3,...batchSize],[u1,u2,u3,...batchSize]...] - * Ref doc : https://docs.sprig.com/reference/post-v2-purge-visitors-1 + * identifierBatches = [[u1,u2,u3,...batchSize],[u1,u2,u3,...batchSize]...] + * Ref doc : https://dev.emarsys.com/docs/core-api-reference/szmq945esac90-delete-contacts */ - const userIdBatches = getUserIdBatches(userAttributes, 100); + const identifierBatches = getCustomIdBatches(userAttributes, configuredPayloadProperty, 1000); // Note: we will only get 400 status code when no user deletion is present for given userIds so we will not throw error in that case // eslint-disable-next-line no-restricted-syntax - for (const curBatch of userIdBatches) { + for (const curBatch of identifierBatches) { + const deleteContactPayload = { + key_id: customIdentifier, + contact_list_id: config.defaultContactList, + }; + deleteContactPayload[`${customIdentifier}`] = curBatch; // eslint-disable-next-line no-await-in-loop const deletionResponse = await httpPOST( endpoint, { - userIds: curBatch, + ...deleteContactPayload, }, { headers, }, { - destType: 'sprig', + destType: 'emarsys', feature: 'deleteUsers', - endpointPath: '/purge/visitors', + endpointPath: '/contact/delete', requestMethod: 'POST', module: 'deletion', }, diff --git a/src/v0/util/deleteUserUtils.js b/src/v0/util/deleteUserUtils.js index 6cf16d7f9e..0e24d15192 100644 --- a/src/v0/util/deleteUserUtils.js +++ b/src/v0/util/deleteUserUtils.js @@ -18,4 +18,16 @@ const getUserIdBatches = (userAttributes, MAX_BATCH_SIZE) => { return userIdBatches; }; -module.exports = { getUserIdBatches }; +const getCustomIdBatches = (userAttributes, customIdentifier, MAX_BATCH_SIZE) => { + const identifierArray = []; + userAttributes.forEach((userAttribute) => { + // Dropping the user if customIdentifier is not present + if (userAttribute[customIdentifier]) { + identifierArray.push(userAttribute.customIdentifier); + } + }); + const identifierBatches = lodash.chunk(identifierArray, MAX_BATCH_SIZE); + return identifierBatches; +}; + +module.exports = { getUserIdBatches, getCustomIdBatches }; diff --git a/src/v1/destinations/emarsys/networkHandler.js b/src/v1/destinations/emarsys/networkHandler.js index 28d2feba30..b31fb40862 100644 --- a/src/v1/destinations/emarsys/networkHandler.js +++ b/src/v1/destinations/emarsys/networkHandler.js @@ -39,7 +39,6 @@ const responseHandler = (responseParams) => { let responseWithIndividualEvents = []; const { response, status } = destinationResponse; - // even if a single event is unsuccessful, the entire batch will fail, we will filter that event out and retry others if (!isHttpStatusSuccess(status)) { const errorMessage = response.replyText || 'unknown error format'; responseWithIndividualEvents = rudderJobMetadata.map((metadata) => ({ From dcde00f95edf7ea31d0e2c77719f4dfd3d9fbb42 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 6 May 2024 16:23:07 +0530 Subject: [PATCH 19/26] fix: small edit --- src/v0/util/deleteUserUtils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v0/util/deleteUserUtils.js b/src/v0/util/deleteUserUtils.js index 0e24d15192..22b5ba6a81 100644 --- a/src/v0/util/deleteUserUtils.js +++ b/src/v0/util/deleteUserUtils.js @@ -23,7 +23,7 @@ const getCustomIdBatches = (userAttributes, customIdentifier, MAX_BATCH_SIZE) => userAttributes.forEach((userAttribute) => { // Dropping the user if customIdentifier is not present if (userAttribute[customIdentifier]) { - identifierArray.push(userAttribute.customIdentifier); + identifierArray.push(userAttribute[customIdentifier]); } }); const identifierBatches = lodash.chunk(identifierArray, MAX_BATCH_SIZE); From 83cd772a7cf24c442e12b8a6623c4c4adebee9d1 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 6 May 2024 18:56:35 +0530 Subject: [PATCH 20/26] fix: edit in network handler --- src/v1/destinations/emarsys/networkHandler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v1/destinations/emarsys/networkHandler.js b/src/v1/destinations/emarsys/networkHandler.js index b31fb40862..120ec6fe37 100644 --- a/src/v1/destinations/emarsys/networkHandler.js +++ b/src/v1/destinations/emarsys/networkHandler.js @@ -10,7 +10,7 @@ const { const tags = require('../../../v0/util/tags'); function checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId) { - const { errors } = destinationResponse.data; + const { errors } = destinationResponse.response.data; // Determine if event is a string or an object, then fetch the corresponding key or value let errorKey; From 1751f4a9f5d02ccb491ab0d172c7c5b5ff3ba01a Mon Sep 17 00:00:00 2001 From: Shrouti Gangopadhyay Date: Tue, 7 May 2024 01:03:18 +0530 Subject: [PATCH 21/26] fix: data delivery test cases finalised --- src/v1/destinations/emarsys/networkHandler.js | 9 +- .../destinations/emarsys/dataDelivery/data.ts | 554 ++++++++++++++---- .../destinations/emarsys/network.ts | 68 ++- 3 files changed, 492 insertions(+), 139 deletions(-) diff --git a/src/v1/destinations/emarsys/networkHandler.js b/src/v1/destinations/emarsys/networkHandler.js index 120ec6fe37..0de2a90edc 100644 --- a/src/v1/destinations/emarsys/networkHandler.js +++ b/src/v1/destinations/emarsys/networkHandler.js @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ +const { isObject } = require('@rudderstack/integrations-lib'); const { TransformerProxyError } = require('../../../v0/util/errorTypes'); const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); const { isHttpStatusSuccess } = require('../../../v0/util/index'); @@ -23,12 +24,18 @@ function checkIfEventIsAbortableAndExtractErrorMessage(event, destinationRespons } // Check if this key has a corresponding error in the errors object - if (errors[errorKey]) { + if (errors && isObject(errors) && errors[errorKey]) { // const errorCode = Object.keys(errors[errorKey])[0]; // Assume there is at least one error code const errorMsg = JSON.stringify(errors[errorKey]); return { isAbortable: true, errorMsg }; } + // if '' is present in the error object, that means, it is a root level error, and none of the events are supposed to be successful + if (errors && isObject(errors) && errors['']) { + const errorMsg = JSON.stringify(errors['']); + return { isAbortable: true, errorMsg }; + } + // Return false and an empty error message if no error is found return { isAbortable: false, errorMsg: '' }; } diff --git a/test/integrations/destinations/emarsys/dataDelivery/data.ts b/test/integrations/destinations/emarsys/dataDelivery/data.ts index cf27c32865..ac3ec780f7 100644 --- a/test/integrations/destinations/emarsys/dataDelivery/data.ts +++ b/test/integrations/destinations/emarsys/dataDelivery/data.ts @@ -1,4 +1,4 @@ -import { generateProxyV1Payload } from '../../../testUtils'; +import { generateMetadata, generateProxyV1Payload } from '../../../testUtils'; import { ProxyV1TestData } from '../../../testTypes'; export const headerBlockWithCorrectAccessToken = { @@ -45,9 +45,9 @@ export const wrongContactCreateUpdateData = [ }, { '2': true, - '3': 1234, - '10569': 'efgh', - '10519': 1234, + '3': 'person0@example.com', + '10569': 1234, + '10519': 'efgh', '31': 2, '39': 'abc', }, @@ -66,16 +66,37 @@ export const correctGroupCallPayload = { export const groupPayloadWithWrongKeyId = { key_id: 'wrong_id', - external_ids: ['efghi', 'jklmn', 'unknown', 'person4@example.com'], + external_ids: ['efghi', 'jklmn'], }; export const groupPayloadWithWrongExternalId = { - key_id: 'wrong_id', + key_id: 'right_id', external_ids: ['efghi', 'jklmn', 'unknown', 'person4@example.com'], }; +export const correctContactWithWrongKeyIdCreateUpdateData = [ + { + '2': 'Person0', + '3': 'person0@example.com', + '10569': 'efghi', + '10519': 'efghi', + '31': 1, + '39': 'abc', + '100': 'abc', + }, + { + '2': true, + '3': 'abcde', + '10569': 'efgh', + '10519': 1234, + '31': 2, + '39': 'abc', + '100': 'abc', + }, +]; + export const statTags = { - destType: 'emarsys', + destType: 'EMARSYS', errorCategory: 'network', destinationId: 'default-destinationId', workspaceId: 'default-workspaceId', @@ -85,90 +106,74 @@ export const statTags = { module: 'destination', }; -export const metadata = { - jobId: 1, - attemptNum: 1, - userId: 'default-userId', - destinationId: 'default-destinationId', - workspaceId: 'default-workspaceId', - sourceId: 'default-sourceId', - secret: { - accessToken: 'default-accessToken', +export const metadata = [ + { + jobId: 1, + attemptNum: 1, + userId: 'default-userId', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + sourceId: 'default-sourceId', + secret: { + accessToken: 'default-accessToken', + }, + dontBatch: false, }, - dontBatch: false, -}; + { + jobId: 2, + attemptNum: 1, + userId: 'default-userId', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + sourceId: 'default-sourceId', + secret: { + accessToken: 'default-accessToken', + }, + dontBatch: false, + }, +]; -// const commonIdentifyRequestParametersWithWrongData = { -// headers: headerBlockWithCorrectAccessToken, -// JSON: { ...contactPayload, contacts: wrongContactCreateUpdateData }, -// }; +const commonIdentifyRequestParametersWithWrongData = { + method: 'PUT', + headers: headerBlockWithCorrectAccessToken, + JSON: { ...contactPayload, contacts: wrongContactCreateUpdateData }, +}; const commonIdentifyRequestParameters = { + method: 'PUT', headers: headerBlockWithCorrectAccessToken, JSON: { ...contactPayload }, }; const commonIdentifyRequestParametersWithWrongKeyId = { + method: 'PUT', headers: headerBlockWithCorrectAccessToken, - JSON: { ...contactPayload, key_id: 100 }, + JSON: { + ...contactPayload, + contacts: correctContactWithWrongKeyIdCreateUpdateData, + key_id: 100, + }, }; -// const commonGroupRequestParametersWithWrongData = { -// headers: headerBlockWithCorrectAccessToken, -// JSON: groupPayloadWithWrongExternalId, -// }; +const commonGroupRequestParametersWithWrongData = { + method: 'POST', + headers: headerBlockWithCorrectAccessToken, + JSON: groupPayloadWithWrongExternalId, +}; -// const commonGroupRequestParameters = { -// headers: headerBlockWithCorrectAccessToken, -// JSON: correctGroupCallPayload, -// }; +const commonGroupRequestParameters = { + method: 'POST', + headers: headerBlockWithCorrectAccessToken, + JSON: correctGroupCallPayload, +}; -// const commonGroupRequestParametersWithWrongKeyId = { -// headers: headerBlockWithCorrectAccessToken, -// JSON: groupPayloadWithWrongKeyId, -// }; +const commonGroupRequestParametersWithWrongKeyId = { + method: 'POST', + headers: headerBlockWithCorrectAccessToken, + JSON: groupPayloadWithWrongKeyId, +}; export const testScenariosForV1API: ProxyV1TestData[] = [ - // { - // id: 'emarsys_v1_scenario_1', - // name: 'emarsys', - // description: 'Identify Event fails due to wrong key_id', - // successCriteria: 'Should return 400 and aborted', - // scenario: 'Business', - // feature: 'dataDelivery', - // module: 'destination', - // version: 'v1', - // input: { - // request: { - // body: generateProxyV1Payload({ - // endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', - // ...commonIdentifyRequestParametersWithWrongKeyId, - // }), - // method: 'PUT', - // }, - // }, - // output: { - // response: { - // status: 200, - // body: { - // output: { - // message: - // "emersys Conversion API: Error transformer proxy v1 during emersys Conversion API response transformation. Incorrect conversions information provided. Conversion's method should be CONVERSIONS_API, indices [0] (0-indexed)", - // response: [ - // { - // error: - // '{"message":"Incorrect conversions information provided. Conversion\'s method should be CONVERSIONS_API, indices [0] (0-indexed)","status":400}', - // statusCode: 400, - // metadata, - // }, - // ], - // statTags, - // status: 400, - // }, - // }, - // }, - // }, - // }, { id: 'emarsys_v1_scenario_1', name: 'emarsys', @@ -180,10 +185,13 @@ export const testScenariosForV1API: ProxyV1TestData[] = [ version: 'v1', input: { request: { - body: generateProxyV1Payload({ - endpoint: `https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1`, - ...commonIdentifyRequestParameters, - }), + body: generateProxyV1Payload( + { + endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', + ...commonIdentifyRequestParametersWithWrongKeyId, + }, + metadata, + ), method: 'POST', }, }, @@ -192,63 +200,361 @@ export const testScenariosForV1API: ProxyV1TestData[] = [ status: 200, body: { output: { - message: - "emersys Conversion API: Error transformer proxy v1 during emersys Conversion API response transformation. Incorrect conversions information provided. Conversion's method should be CONVERSIONS_API, indices [0] (0-indexed)", + status: 200, + message: '[EMARSYS Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + status: 200, + data: { + ids: [], + errors: { + '': { + '2004': 'Invalid key field id: 100', + }, + }, + }, + }, + status: 200, + }, response: [ { - error: - '{"message":"Incorrect conversions information provided. Conversion\'s method should be CONVERSIONS_API, indices [0] (0-indexed)","status":400}', statusCode: 400, - metadata, + metadata: generateMetadata(1), + error: '{"2004":"Invalid key field id: 100"}', + }, + { + statusCode: 400, + metadata: generateMetadata(2), + error: '{"2004":"Invalid key field id: 100"}', + }, + ], + }, + }, + }, + }, + }, + { + id: 'emarsys_v1_scenario_1', + name: 'emarsys', + description: 'correct Identify event passes with 200 status code', + successCriteria: 'Should return 400 and aborted', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: `https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1`, + ...commonIdentifyRequestParameters, + }, + metadata, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[EMARSYS Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + replyCode: 0, + replyText: 'OK', + data: { ids: ['138621551', 968984932] }, + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + { + statusCode: 200, + metadata: generateMetadata(2), + error: 'success', + }, + ], + }, + }, + }, + }, + }, + { + id: 'emarsys_v1_scenario_1', + name: 'emarsys', + description: 'Identify Event fails due to wrong data', + successCriteria: 'Should return 400 and aborted', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: `https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1`, + ...commonIdentifyRequestParametersWithWrongData, + }, + metadata, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[EMARSYS Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + status: 200, + data: { + ids: ['138621551'], + errors: { + '1234': { + '2010': 'Contacts with the external id already exist: 3', + }, + }, + }, + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + { + statusCode: 400, + metadata: generateMetadata(2), + error: '{"2010":"Contacts with the external id already exist: 3"}', + }, + ], + }, + }, + }, + }, + }, + { + id: 'emarsys_v1_scenario_1', + name: 'emarsys', + description: 'correct Group event passes with 200 status code', + successCriteria: 'Should return 400 and aborted', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: `https://api.emarsys.net/api/v2/contactlist/900337462/add`, + ...commonGroupRequestParameters, + }, + [generateMetadata(1)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[EMARSYS Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + replyCode: 0, + replyText: 'OK', + data: { errors: [], inserted_contacts: 1 }, + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', }, ], + }, + }, + }, + }, + }, + { + id: 'emarsys_v1_scenario_1', + name: 'emarsys', + description: 'Group Event fails due to wrong key_id', + successCriteria: 'Should return 400 and aborted', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: `https://api.emarsys.net/api/v2/contactlist/900337462/add`, + ...commonGroupRequestParametersWithWrongKeyId, + }, + metadata, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 400, statTags, + message: + 'EMARSYS: Error transformer proxy v1 during EMARSYS response transformation. Invalid key field id: wrong_id', + response: [ + { + statusCode: 400, + metadata: generateMetadata(1), + error: '{"replyCode":2004,"replyText":"Invalid key field id: wrong_id","data":""}', + }, + { + statusCode: 400, + metadata: generateMetadata(2), + error: '{"replyCode":2004,"replyText":"Invalid key field id: wrong_id","data":""}', + }, + ], + }, + }, + }, + }, + }, + { + id: 'emarsys_v1_scenario_1', + name: 'emarsys', + description: 'Group Event fails due to wrong data', + successCriteria: 'Should return 400 and aborted', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: `https://api.emarsys.net/api/v2/contactlist/900337462/add`, + ...commonGroupRequestParametersWithWrongData, + }, + [generateMetadata(1), generateMetadata(2), generateMetadata(3), generateMetadata(4)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[EMARSYS Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + replyCode: 0, + replyText: 'OK', + data: { + inserted_contacts: 2, + errors: { + jklmn: { + '2008': 'No contact found with the external id: 3', + }, + unknown: { + '2008': 'No contact found with the external id: 3', + }, + }, + }, + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + { + statusCode: 400, + metadata: generateMetadata(2), + error: '{"2008":"No contact found with the external id: 3"}', + }, + { + statusCode: 400, + metadata: generateMetadata(3), + error: '{"2008":"No contact found with the external id: 3"}', + }, + { + statusCode: 200, + metadata: generateMetadata(4), + error: 'success', + }, + ], + }, + }, + }, + }, + }, + { + id: 'emarsys_v1_scenario_1', + name: 'emarsys', + description: 'Group Event fails due to wrong contact list id', + successCriteria: 'Should return 400 and aborted', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.emarsys.net/api/v2/contactlist/wrong-id/add', + ...commonGroupRequestParameters, + }, + [generateMetadata(1)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { status: 400, + statTags, + message: + 'EMARSYS: Error transformer proxy v1 during EMARSYS response transformation. Action Wrong-id is invalid.', + response: [ + { + statusCode: 400, + metadata: generateMetadata(1), + error: '{"replyCode":1,"replyText":"Action Wrong-id is invalid.","data":""}', + }, + ], }, }, }, }, }, - // { - // id: 'emarsys_v1_scenario_1', - // name: 'emarsys', - // description: 'Identify Event fails due to wrong key_id', - // successCriteria: 'Should return 400 and aborted', - // scenario: 'Business', - // feature: 'dataDelivery', - // module: 'destination', - // version: 'v1', - // input: { - // request: { - // body: generateProxyV1Payload({ - // endpoint: `https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1`, - // ...commonIdentifyRequestParametersWithWrongData, - // }), - // method: 'POST', - // }, - // }, - // output: { - // response: { - // status: 200, - // body: { - // output: { - // message: - // "emersys Conversion API: Error transformer proxy v1 during emersys Conversion API response transformation. Incorrect conversions information provided. Conversion's method should be CONVERSIONS_API, indices [0] (0-indexed)", - // response: [ - // { - // error: - // '{"message":"Incorrect conversions information provided. Conversion\'s method should be CONVERSIONS_API, indices [0] (0-indexed)","status":400}', - // statusCode: 400, - // metadata, - // }, - // ], - // statTags, - // status: 400, - // }, - // }, - // }, - // }, - // }, ]; export const data = [...testScenariosForV1API]; diff --git a/test/integrations/destinations/emarsys/network.ts b/test/integrations/destinations/emarsys/network.ts index 6fab8b8c38..75004de886 100644 --- a/test/integrations/destinations/emarsys/network.ts +++ b/test/integrations/destinations/emarsys/network.ts @@ -31,7 +31,7 @@ export const correctContactCreateUpdateData = [ }, ]; -export const wrongContactCreateUpdateData = [ +export const correctContactWithWrongKeyIdCreateUpdateData = [ { '2': 'Person0', '3': 'person0@example.com', @@ -39,14 +39,35 @@ export const wrongContactCreateUpdateData = [ '10519': 'efghi', '31': 1, '39': 'abc', + '100': 'abc', }, { '2': true, - '3': 1234, + '3': 'abcde', '10569': 'efgh', '10519': 1234, '31': 2, '39': 'abc', + '100': 'abc', + }, +]; + +export const wrongContactCreateUpdateData = [ + { + '2': 'Person0', + '3': 'person0@example.com', + '10569': 'efghi', + '10519': 'efghi', + '31': 1, + '39': 'abc', + }, + { + '2': true, + '3': 'person0@example.com', + '10569': 1234, + '10519': 'efgh', + '31': 2, + '39': 'abc', }, ]; @@ -63,11 +84,11 @@ export const correctGroupCallPayload = { export const groupPayloadWithWrongKeyId = { key_id: 'wrong_id', - external_ids: ['efghi', 'jklmn', 'unknown', 'person4@example.com'], + external_ids: ['efghi', 'jklmn'], }; export const groupPayloadWithWrongExternalId = { - key_id: 'wrong_id', + key_id: 'right_id', external_ids: ['efghi', 'jklmn', 'unknown', 'person4@example.com'], }; @@ -81,7 +102,15 @@ const businessMockData = [ headers: headerBlockWithCorrectAccessToken, data: contactPayload, }, - httpRes: { replyCode: 0, replyText: 'OK', data: { ids: ['138621551', 968984932] } }, + httpRes: { + data: { + replyCode: 0, + replyText: 'OK', + data: { ids: ['138621551', 968984932] }, + }, + status: 200, + statusText: 'OK', + }, }, { description: @@ -94,10 +123,14 @@ const businessMockData = [ }, httpRes: { data: { - ids: ['138621551'], - errors: { '1234': { '2010': 'Contacts with the external id already exist: 3' } }, + data: { + ids: ['138621551'], + errors: { '1234': { '2010': 'Contacts with the external id already exist: 3' } }, + }, + status: 200, }, status: 200, + statusText: 'OK', }, }, { @@ -106,17 +139,25 @@ const businessMockData = [ method: 'PUT', url: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', headers: headerBlockWithCorrectAccessToken, - data: { ...contactPayload, key_id: 100 }, + data: { + ...contactPayload, + contacts: correctContactWithWrongKeyIdCreateUpdateData, + key_id: 100, + }, }, httpRes: { - data: { ids: [], errors: { '': { '2004': 'Invalid key field id: 100' } } }, + data: { + data: { ids: [], errors: { '': { '2004': 'Invalid key field id: 100' } } }, + status: 200, + }, status: 200, + statusText: 'OK', }, }, { description: 'Mock response from destination for correct group call ', httpReq: { - method: 'put', + method: 'POST', url: 'https://api.emarsys.net/api/v2/contactlist/900337462/add', headers: headerBlockWithCorrectAccessToken, data: correctGroupCallPayload, @@ -135,12 +176,12 @@ const businessMockData = [ data: groupPayloadWithWrongKeyId, }, httpRes: { - data: { replyCode: 2004, replyText: 'Invalid key field id: 100', data: '' }, + data: { replyCode: 2004, replyText: 'Invalid key field id: wrong_id', data: '' }, status: 400, }, }, { - description: 'Mock response from destination for group call with wrong key_id ', + description: 'Mock response from destination for group call with wrong data ', httpReq: { method: 'POST', url: 'https://api.emarsys.net/api/v2/contactlist/900337462/add', @@ -152,9 +193,8 @@ const businessMockData = [ replyCode: 0, replyText: 'OK', data: { - inserted_contacts: 0, + inserted_contacts: 2, errors: { - efghi: { '2008': 'No contact found with the external id: 3' }, jklmn: { '2008': 'No contact found with the external id: 3' }, unknown: { '2008': 'No contact found with the external id: 3' }, }, From 3694148794e9587ca61c4f1c8d3375ec3be5e1b0 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Tue, 7 May 2024 12:50:39 +0530 Subject: [PATCH 22/26] fix: editing unit test cases --- src/cdk/v2/destinations/emarsys/utils.js | 2 +- src/cdk/v2/destinations/emarsys/utils.test.js | 133 ++++++++++-------- 2 files changed, 73 insertions(+), 62 deletions(-) diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index aa551124ae..9059f35806 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -64,7 +64,7 @@ const buildIdentifyPayload = (message, destConfig) => { const integrationObject = getIntegrationsObj(message, 'emarsys'); const finalContactList = integrationObject?.contactListId || defaultContactList; - if (!isDefinedAndNotNullAndNotEmpty(String(finalContactList))) { + if (!finalContactList || !isDefinedAndNotNullAndNotEmpty(String(finalContactList))) { throw new InstrumentationError( 'Cannot a find a specific contact list either through configuration or via integrations object', ); diff --git a/src/cdk/v2/destinations/emarsys/utils.test.js b/src/cdk/v2/destinations/emarsys/utils.test.js index 15147d9cdd..991f7fe6b1 100644 --- a/src/cdk/v2/destinations/emarsys/utils.test.js +++ b/src/cdk/v2/destinations/emarsys/utils.test.js @@ -12,6 +12,7 @@ const { } = require('../../../../v1/destinations/emarsys/networkHandler'); const crypto = require('crypto'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { responses } = require('../../../../../test/testHelper'); describe('base64Sha', () => { it('should return a base64 encoded SHA1 hash of the input string', () => { @@ -214,7 +215,7 @@ describe('buildGroupPayload', () => { eventType: 'group', destinationPayload: { payload: { - key_id: '3', + key_id: 3, external_ids: ['test@example.com'], }, contactListId: 'group123', @@ -234,14 +235,12 @@ describe('buildGroupPayload', () => { }, }; const destination = { - Config: { - emersysCustomIdentifier: '100', - defaultContactList: 'list123', - fieldMapping: [ - { emersysProperty: '100', rudderProperty: 'customId' }, - { emersysProperty: '3', rudderProperty: 'email' }, - ], - }, + emersysCustomIdentifier: '100', + defaultContactList: 'list123', + fieldMapping: [ + { emersysProperty: '100', rudderProperty: 'customId' }, + { emersysProperty: '3', rudderProperty: 'email' }, + ], }; const result = buildGroupPayload(message, destination); expect(result).toEqual({ @@ -267,14 +266,12 @@ describe('buildGroupPayload', () => { }, }; const destination = { - Config: { - emersysCustomIdentifier: 'customId', - defaultContactList: 'list123', - fieldMapping: [ - { emersysProperty: 'customId', rudderProperty: 'customId' }, - { emersysProperty: 'email', rudderProperty: 'email' }, - ], - }, + emersysCustomIdentifier: 'customId', + defaultContactList: 'list123', + fieldMapping: [ + { emersysProperty: 'customId', rudderProperty: 'customId' }, + { emersysProperty: 'email', rudderProperty: 'email' }, + ], }; expect(() => { buildGroupPayload(message, destination); @@ -288,51 +285,57 @@ describe('createGroupBatches', () => { // Arrange const events = [ { - message: { - body: { - JSON: { - destinationPayload: { - payload: { - key_id: 'key1', - external_ids: ['id1', 'id2'], + message: [ + { + body: { + JSON: { + destinationPayload: { + payload: { + key_id: 'key1', + external_ids: ['id1', 'id2'], + }, + contactListId: 'list1', }, - contactListId: 'list1', }, }, }, - }, + ], metadata: { jobId: 1, userId: 'u1' }, }, { - message: { - body: { - JSON: { - destinationPayload: { - payload: { - key_id: 'key2', - external_ids: ['id3', 'id4'], + message: [ + { + body: { + JSON: { + destinationPayload: { + payload: { + key_id: 'key2', + external_ids: ['id3', 'id4'], + }, + contactListId: 'list2', }, - contactListId: 'list2', }, }, }, - }, + ], metadata: { jobId: 2, userId: 'u2' }, }, { - message: { - body: { - JSON: { - destinationPayload: { - payload: { - key_id: 'key1', - external_ids: ['id5', 'id6'], + message: [ + { + body: { + JSON: { + destinationPayload: { + payload: { + key_id: 'key1', + external_ids: ['id5', 'id6'], + }, + contactListId: 'list1', }, - contactListId: 'list1', }, }, }, - }, + ], metadata: { jobId: 3, userId: 'u3' }, }, ]; @@ -380,7 +383,7 @@ describe('createGroupBatches', () => { describe('findRudderPropertyByEmersysProperty', () => { // Returns the correct rudderProperty when given a valid emersysProperty and fieldMapping it('should return the correct rudderProperty when given a valid emersysProperty and fieldMapping', () => { - const emersysProperty = 'email'; + const emersysProperty = 'firstName'; const fieldMapping = [ { emersysProperty: 'email', rudderProperty: 'email' }, { emersysProperty: 'firstName', rudderProperty: 'firstName' }, @@ -389,7 +392,7 @@ describe('findRudderPropertyByEmersysProperty', () => { const result = findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping); - expect(result).toBe('email'); + expect(result).toBe('firstName'); }); // Returns null when given an empty fieldMapping @@ -399,7 +402,7 @@ describe('findRudderPropertyByEmersysProperty', () => { const result = findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping); - expect(result).toBeNull(); + expect(result).toBe('email'); }); }); @@ -408,10 +411,12 @@ describe('checkIfEventIsAbortableAndExtractErrorMessage', () => { it('should return {isAbortable: false, errorMsg: ""} when event is neither a string nor an object with keyId', () => { const event = 123; const destinationResponse = { - data: { - errors: { - errorKey: { - errorCode: 'errorMessage', + response: { + data: { + errors: { + errorKey: { + errorCode: 'errorMessage', + }, }, }, }, @@ -427,8 +432,10 @@ describe('checkIfEventIsAbortableAndExtractErrorMessage', () => { it('should return {isAbortable: false, errorMsg: ""} when errors object is empty', () => { const event = 'event'; const destinationResponse = { - data: { - errors: {}, + response: { + data: { + errors: {}, + }, }, }; const keyId = 'keyId'; @@ -442,10 +449,12 @@ describe('checkIfEventIsAbortableAndExtractErrorMessage', () => { it('should return {isAbortable: true, errorMsg} when event is a string and has a corresponding error in the errors object', () => { const event = 'event'; const destinationResponse = { - data: { - errors: { - event: { - errorCode: 'errorMessage', + response: { + data: { + errors: { + event: { + errorCode: 'errorMessage', + }, }, }, }, @@ -463,10 +472,12 @@ describe('checkIfEventIsAbortableAndExtractErrorMessage', () => { keyId: 'event', }; const destinationResponse = { - data: { - errors: { - event: { - errorCode: 'errorMessage', + response: { + data: { + errors: { + event: { + errorCode: 'errorMessage', + }, }, }, }, From 5a058bffa70a52b86a576d0a830b5d06bac8cbd4 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Tue, 7 May 2024 20:20:19 +0530 Subject: [PATCH 23/26] fix: resolve fake timer clash --- .../destinations/emarsys/processor/data.ts | 73 +++++-------------- test/integrations/testUtils.ts | 3 - 2 files changed, 18 insertions(+), 58 deletions(-) diff --git a/test/integrations/destinations/emarsys/processor/data.ts b/test/integrations/destinations/emarsys/processor/data.ts index f3181f3d81..fbeca6f4d8 100644 --- a/test/integrations/destinations/emarsys/processor/data.ts +++ b/test/integrations/destinations/emarsys/processor/data.ts @@ -1,12 +1,20 @@ export const mockFns = (_) => { // @ts-ignore - jest.useFakeTimers().setSystemTime(new Date('2019-10-14')); + jest.useFakeTimers().setSystemTime(new Date('2023-10-14')); jest.mock('crypto', () => ({ ...jest.requireActual('crypto'), randomBytes: jest.fn().mockReturnValue(Buffer.from('5398e214ae99c2e50afb709a3bc423f9', 'hex')), })); }; +const comonHeader = { + Accept: 'application/json', + 'Content-Type': 'application/json', + + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="MjEzMDY5ZmI3NjMwNzE1N2M1ZTI5MWMzMzI3ODQxNDU2YWM4NTI3YQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2023-10-14T00:00:00.000Z"', +}; + export const data = [ { name: 'emarsys', @@ -123,12 +131,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: 'https://api.emarsys.net/api/v2/event/purchase/trigger', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', - }, + headers: comonHeader, params: {}, body: { JSON: { @@ -301,12 +304,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: 'https://api.emarsys.net/api/v2/event/purchase/trigger', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', - }, + headers: comonHeader, params: {}, body: { JSON: { @@ -477,12 +475,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: 'https://api.emarsys.net/api/v2/event/purchase/trigger', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', - }, + headers: comonHeader, params: {}, body: { JSON: { @@ -625,12 +618,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: 'https://api.emarsys.net/api/v2/contactlist/dummy/add', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', - }, + headers: comonHeader, params: {}, body: { JSON: { @@ -753,12 +741,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: 'https://api.emarsys.net/api/v2/contactlist/dummy/add', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', - }, + headers: comonHeader, params: {}, body: { JSON: { @@ -878,12 +861,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: 'https://api.emarsys.net/api/v2/contactlist/dummy/add', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', - }, + headers: comonHeader, params: {}, body: { JSON: { @@ -1005,12 +983,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', - }, + headers: comonHeader, params: {}, body: { JSON: { @@ -1139,12 +1112,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', - }, + headers: comonHeader, params: {}, body: { JSON: { @@ -1271,12 +1239,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: 'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - 'X-WSSE': - 'UsernameToken Username="dummy", PasswordDigest="NDc5MjNlODIyMGE4ODhiMTQyNTA0OGMzZTFjZTM1MmMzMmU0NmNiNw==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2019-10-14T00:00:00.000Z"', - }, + headers: comonHeader, params: {}, body: { JSON: { diff --git a/test/integrations/testUtils.ts b/test/integrations/testUtils.ts index eacdce8b39..0a2727f4d0 100644 --- a/test/integrations/testUtils.ts +++ b/test/integrations/testUtils.ts @@ -22,13 +22,10 @@ import { const generateAlphanumericId = (size = 36) => [...Array(size)].map(() => ((Math.random() * size) | 0).toString(size)).join(''); export const getTestDataFilePaths = (dirPath: string, opts: OptionValues): string[] => { - console.log('dirPath', dirPath); const globPattern = join(dirPath, '**', 'data.ts'); let testFilePaths = globSync(globPattern); - // console.log('testFilePaths', JSON.stringify(testFilePaths)); if (opts.destination) { testFilePaths = testFilePaths.filter((testFile) => testFile.includes(opts.destination)); - console.log('testFilePaths', JSON.stringify(testFilePaths)); } if (opts.feature) { testFilePaths = testFilePaths.filter((testFile) => testFile.includes(opts.feature)); From 1452ed4e69e3b8163e42e12ef8eb29bc580f8e1d Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Wed, 8 May 2024 00:38:55 +0530 Subject: [PATCH 24/26] fix: add user deletion test cases --- src/v0/destinations/emarsys/deleteUsers.js | 9 +- .../destinations/emarsys/deleteUsers/data.ts | 235 ++++++++++++++++++ .../destinations/emarsys/network.ts | 79 +++++- 3 files changed, 321 insertions(+), 2 deletions(-) create mode 100644 test/integrations/destinations/emarsys/deleteUsers/data.ts diff --git a/src/v0/destinations/emarsys/deleteUsers.js b/src/v0/destinations/emarsys/deleteUsers.js index 42a1bd4d8a..c6ca746217 100644 --- a/src/v0/destinations/emarsys/deleteUsers.js +++ b/src/v0/destinations/emarsys/deleteUsers.js @@ -1,4 +1,8 @@ -const { NetworkError } = require('@rudderstack/integrations-lib'); +const { + NetworkError, + isDefinedAndNotNull, + ConfigurationAuthError, +} = require('@rudderstack/integrations-lib'); const { httpPOST } = require('../../../adapters/network'); const { processAxiosResponse, @@ -28,6 +32,9 @@ const userDeletionHandler = async (userAttributes, config) => { customIdentifier, config.fieldMapping, ); + if (!isDefinedAndNotNull(config.defaultContactList)) { + throw new ConfigurationAuthError('No audience list is configured. Aborting'); + } /** * identifierBatches = [[u1,u2,u3,...batchSize],[u1,u2,u3,...batchSize]...] * Ref doc : https://dev.emarsys.com/docs/core-api-reference/szmq945esac90-delete-contacts diff --git a/test/integrations/destinations/emarsys/deleteUsers/data.ts b/test/integrations/destinations/emarsys/deleteUsers/data.ts new file mode 100644 index 0000000000..2bafe58a4c --- /dev/null +++ b/test/integrations/destinations/emarsys/deleteUsers/data.ts @@ -0,0 +1,235 @@ +export const mockFns = (_) => { + // @ts-ignore + jest.useFakeTimers().setSystemTime(new Date('2023-10-14')); + jest.mock('crypto', () => ({ + ...jest.requireActual('crypto'), + randomBytes: jest.fn().mockReturnValue(Buffer.from('5398e214ae99c2e50afb709a3bc423f9', 'hex')), + })); +}; + +const commonEventMap = [ + { + from: 'Order Completed', + to: 'purchase', + }, + { + from: 'Order Completed', + to: 'purchase', + }, +]; + +const commonFieldMap = [ + { + rudderProperty: 'email', + emersysProperty: '3', + }, + { + rudderProperty: 'lastName', + emersysProperty: '2', + }, + { + rudderProperty: 'firstName', + emersysProperty: '1', + }, + { + rudderProperty: 'custom-field', + emersysProperty: 'custom_id', + }, +]; + +export const data = [ + { + name: 'emarsys', + description: 'Missing emersysUsername key', + feature: 'userDeletion', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destType: 'EMARSYS', + userAttributes: [ + { + userId: '1234', + phone: '1234567890', + email: 'abc@xyc.com', + }, + ], + config: { + discardEmptyProperties: true, + emersysUsername: undefined, + emersysUserSecret: 'dummySecret', + emersysCustomIdentifier: '', + defaultContactList: 'dummy', + eventsMapping: commonEventMap, + fieldMapping: commonFieldMap, + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + ], + }, + }, + output: { + response: { + status: 400, + body: [ + { + statusCode: 400, + error: 'Either Emarsys user name or user secret is missing. Aborting', + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'Default contact list is not configured', + feature: 'userDeletion', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destType: 'EMARSYS', + userAttributes: [ + { + userId: '1234', + phone: '1234567890', + email: 'abc@xyc.com', + lastName: 'doe', + }, + ], + config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '2', + defaultContactList: undefined, + eventsMapping: commonEventMap, + fieldMapping: commonFieldMap, + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + ], + }, + }, + output: { + response: { + status: 400, + body: [ + { + statusCode: 400, + error: 'No audience list is configured. Aborting', + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'custom identifier is not present in user attribute', + feature: 'userDeletion', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destType: 'EMARSYS', + userAttributes: [ + { + userId: '1234', + phone: '1234567890', + lastName: 'doe', + }, + ], + config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '', + defaultContactList: 'dummy', + eventsMapping: commonEventMap, + fieldMapping: commonFieldMap, + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 200, + status: 'successful', + }, + ], + }, + }, + }, + { + name: 'emarsys', + description: 'user not present for deletion', + feature: 'userDeletion', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destType: 'EMARSYS', + userAttributes: [ + { + userId: '1234', + email: 'abc@gmail.com', + phone: '1234567890', + lastName: 'doe', + }, + ], + config: { + discardEmptyProperties: true, + emersysUsername: 'dummy', + emersysUserSecret: 'dummy', + emersysCustomIdentifier: '', + defaultContactList: 'dummy', + eventsMapping: commonEventMap, + fieldMapping: commonFieldMap, + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 200, + status: 'successful', + }, + ], + }, + }, + }, +].map((d) => ({ ...d, mockFns })); diff --git a/test/integrations/destinations/emarsys/network.ts b/test/integrations/destinations/emarsys/network.ts index 75004de886..c4954afd91 100644 --- a/test/integrations/destinations/emarsys/network.ts +++ b/test/integrations/destinations/emarsys/network.ts @@ -92,6 +92,14 @@ export const groupPayloadWithWrongExternalId = { external_ids: ['efghi', 'jklmn', 'unknown', 'person4@example.com'], }; +export const comonHeader = { + Accept: 'application/json', + 'Content-Type': 'application/json', + + 'X-WSSE': + 'UsernameToken Username="dummy", PasswordDigest="MjEzMDY5ZmI3NjMwNzE1N2M1ZTI5MWMzMzI3ODQxNDU2YWM4NTI3YQ==", Nonce="5398e214ae99c2e50afb709a3bc423f9", Created="2023-10-14T00:00:00.000Z"', +}; + // MOCK DATA const businessMockData = [ { @@ -218,4 +226,73 @@ const businessMockData = [ }, ]; -export const networkCallsData = [...businessMockData]; +const deleteNwData = [ + { + httpReq: { + method: 'post', + url: 'https://api.emarsys.net/api/v2/contact/delete', + data: { + key_id: 3, + contact_list_id: 'dummy', + 3: ['abc@gmail.com'], + }, + headers: comonHeader, + }, + httpRes: { + data: { + replyCode: 2008, + replyText: 'No contact found with the external id: 3 - abc@gmail.com', + data: '', + }, + status: 200, + }, + }, + { + httpReq: { + method: 'post', + url: 'https://api.emarsys.net/api/v2/contact/delete', + data: { + userIds: ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'], + }, + headers: comonHeader, + }, + httpRes: { + data: 'Your application has made too many requests in too short a time.', + status: 429, + }, + }, + { + httpReq: { + method: 'post', + url: 'https://api.emarsys.net/api/v2/contact/delete', + data: { + userIds: ['9'], + }, + headers: comonHeader, + }, + httpRes: { + data: { + error: 'User deletion request failed', + }, + status: 400, + }, + }, + { + httpReq: { + method: 'post', + url: 'https://api.emarsys.net/api/v2/contact/delete', + data: { + userIds: ['1', '2', '3'], + }, + headers: comonHeader, + }, + httpRes: { + data: { + requestId: 'request_1', + }, + status: 200, + }, + }, +]; + +export const networkCallsData = [...businessMockData, ...deleteNwData]; From b4914351e8322e6cb5bbdadd04a0fc9d6dbbe088 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Fri, 10 May 2024 16:21:08 +0530 Subject: [PATCH 25/26] fix: doc links added --- src/cdk/v2/destinations/emarsys/procWorkflow.yaml | 2 +- src/v1/destinations/emarsys/networkHandler.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cdk/v2/destinations/emarsys/procWorkflow.yaml b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml index 0d3152767e..176ddb71ba 100644 --- a/src/cdk/v2/destinations/emarsys/procWorkflow.yaml +++ b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml @@ -44,7 +44,7 @@ steps: - name: preparePayloadForIdentify description: | Builds identify payload. - ref: + ref: https://dev.emarsys.com/docs/core-api-reference/f8ljhut3ac2i1-update-contacts condition: $.outputs.messageType === {{$.EventType.IDENTIFY}} template: | $.context.payload = $.buildIdentifyPayload(.message, .destination.Config,); diff --git a/src/v1/destinations/emarsys/networkHandler.js b/src/v1/destinations/emarsys/networkHandler.js index 0de2a90edc..c88d2b02dd 100644 --- a/src/v1/destinations/emarsys/networkHandler.js +++ b/src/v1/destinations/emarsys/networkHandler.js @@ -10,6 +10,7 @@ const { } = require('../../../adapters/utils/networkUtils'); const tags = require('../../../v0/util/tags'); +// ref : https://dev.emarsys.com/docs/emarsys-core-api-guides/c47a64a8ea7dc-http-200-errors function checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId) { const { errors } = destinationResponse.response.data; @@ -46,6 +47,7 @@ const responseHandler = (responseParams) => { let responseWithIndividualEvents = []; const { response, status } = destinationResponse; + // ref : https://dev.emarsys.com/docs/emarsys-core-api-guides/5e68295991665-http-400-errors if (!isHttpStatusSuccess(status)) { const errorMessage = response.replyText || 'unknown error format'; responseWithIndividualEvents = rudderJobMetadata.map((metadata) => ({ From 305e2314d866060946344b22a78c71fee71f81cb Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Mon, 13 May 2024 12:55:52 +0530 Subject: [PATCH 26/26] fix: review comments addressed --- .../v2/destinations/emarsys/procWorkflow.yaml | 15 +- src/cdk/v2/destinations/emarsys/utils.js | 33 - src/cdk/v2/destinations/emarsys/utils.test.js | 838 ++++++++++-------- src/v1/destinations/emarsys/networkHandler.js | 19 +- 4 files changed, 460 insertions(+), 445 deletions(-) diff --git a/src/cdk/v2/destinations/emarsys/procWorkflow.yaml b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml index 176ddb71ba..a5c0b33f38 100644 --- a/src/cdk/v2/destinations/emarsys/procWorkflow.yaml +++ b/src/cdk/v2/destinations/emarsys/procWorkflow.yaml @@ -13,6 +13,8 @@ bindings: path: ../../../../v0/util - name: getFieldValueFromMessage path: ../../../../v0/util + - name: CommonUtils + path: ../../../../util/common - path: ./utils - path: ./config - path: lodash @@ -43,22 +45,19 @@ steps: $.assert(.message.event, "event could not be mapped to conversion rule. Aborting.") - name: preparePayloadForIdentify description: | - Builds identify payload. - ref: https://dev.emarsys.com/docs/core-api-reference/f8ljhut3ac2i1-update-contacts + Builds identify payload. ref: https://dev.emarsys.com/docs/core-api-reference/f8ljhut3ac2i1-update-contacts condition: $.outputs.messageType === {{$.EventType.IDENTIFY}} template: | $.context.payload = $.buildIdentifyPayload(.message, .destination.Config,); - name: preparePayloadForGroup description: | - Builds group payload. - ref: https://dev.emarsys.com/docs/core-api-reference/1m0m70hy3tuov-add-contacts-to-a-contact-list + Builds group payload. ref: https://dev.emarsys.com/docs/core-api-reference/1m0m70hy3tuov-add-contacts-to-a-contact-list condition: $.outputs.messageType === {{$.EventType.GROUP}} template: | $.context.payload = $.buildGroupPayload(.message, .destination.Config,); - name: preparePayloadForTrack description: | - Builds track payload. - ref: https://dev.emarsys.com/docs/core-api-reference/fl0xx6rwfbwqb-trigger-an-external-event + Builds track payload. ref: https://dev.emarsys.com/docs/core-api-reference/fl0xx6rwfbwqb-trigger-an-external-event condition: $.outputs.messageType === {{$.EventType.TRACK}} template: | const properties = ^.message.properties; @@ -69,9 +68,7 @@ steps: external_id: $.deduceExternalIdValue(^.message,emersysIdentifierId,.destination.Config.fieldMapping), trigger_id: integrationObject.trigger_id, data: properties.data, - attachment: Array.isArray(properties.attachment) - ? properties.attachment - : [properties.attachment], + attachment:$.CommonUtils.toArray(properties.attachment), event_time: $.getFieldValueFromMessage(^.message, 'timestamp'), }; $.context.payload = { diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js index 9059f35806..2fe686718d 100644 --- a/src/cdk/v2/destinations/emarsys/utils.js +++ b/src/cdk/v2/destinations/emarsys/utils.js @@ -12,7 +12,6 @@ const { getIntegrationsObj, validateEventName, getValueFromMessage, - getFieldValueFromMessage, getHashFromArray, } = require('../../../../v0/util'); const { @@ -173,37 +172,6 @@ const deduceEventId = (message, destConfig) => { return eventId; }; -const buildTrackPayload = (message, destination) => { - let eventId; - const { emersysCustomIdentifier, eventsMapping } = destination.Config; - const { event, properties } = message; - - if (eventsMapping.length > 0) { - const keyMap = getHashFromArray(eventsMapping, 'from', 'to', false); - eventId = keyMap[event]; - } - const integrationObject = getIntegrationsObj(message, 'emersys'); - const emersysIdentifier = - integrationObject?.customIdentifierId || emersysCustomIdentifier || EMAIL_FIELD_ID; - const payload = { - key_id: emersysIdentifier, - external_id: 'test@example.com', - trigger_id: integrationObject.trigger_id, - data: properties.data, - attachment: Array.isArray(properties.attatchment) - ? properties.attatchment - : [properties.attatchment], - event_time: getFieldValueFromMessage(message, 'timestamp'), - }; - return { - eventType: message.type, - destinationPayload: { - payload: removeUndefinedAndNullValues(payload), - event: eventId, - }, - }; -}; - const deduceEndPoint = (finalPayload) => { let endPoint; let eventId; @@ -437,7 +405,6 @@ module.exports = { createIdentifyBatches, ensureSizeConstraints, createGroupBatches, - buildTrackPayload, deduceExternalIdValue, deduceEventId, deduceCustomIdentifier, diff --git a/src/cdk/v2/destinations/emarsys/utils.test.js b/src/cdk/v2/destinations/emarsys/utils.test.js index 991f7fe6b1..3802567ecb 100644 --- a/src/cdk/v2/destinations/emarsys/utils.test.js +++ b/src/cdk/v2/destinations/emarsys/utils.test.js @@ -6,486 +6,538 @@ const { getWsseHeader, findRudderPropertyByEmersysProperty, createGroupBatches, + deduceEventId, } = require('./utils'); const { checkIfEventIsAbortableAndExtractErrorMessage, } = require('../../../../v1/destinations/emarsys/networkHandler'); const crypto = require('crypto'); -const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); const { responses } = require('../../../../../test/testHelper'); -describe('base64Sha', () => { - it('should return a base64 encoded SHA1 hash of the input string', () => { - const input = 'test'; - const expected = 'YTk0YThmZTVjY2IxOWJhNjFjNGMwODczZDM5MWU5ODc5ODJmYmJkMw=='; - const result = base64Sha(input); - expect(result).toEqual(expected); - }); +describe('Emarsys utils', () => { + describe('base64Sha', () => { + it('should return a base64 encoded SHA1 hash of the input string', () => { + const input = 'test'; + const expected = 'YTk0YThmZTVjY2IxOWJhNjFjNGMwODczZDM5MWU5ODc5ODJmYmJkMw=='; + const result = base64Sha(input); + expect(result).toEqual(expected); + }); - it('should return an empty string when input is empty', () => { - const input = ''; - const expected = 'ZGEzOWEzZWU1ZTZiNGIwZDMyNTViZmVmOTU2MDE4OTBhZmQ4MDcwOQ=='; - const result = base64Sha(input); - expect(result).toEqual(expected); + it('should return an empty string when input is empty', () => { + const input = ''; + const expected = 'ZGEzOWEzZWU1ZTZiNGIwZDMyNTViZmVmOTU2MDE4OTBhZmQ4MDcwOQ=='; + const result = base64Sha(input); + expect(result).toEqual(expected); + }); }); -}); -describe('getWsseHeader', () => { - beforeEach(() => { - jest - .spyOn(crypto, 'randomBytes') - .mockReturnValue(Buffer.from('abcdef1234567890abcdef1234567890', 'hex')); - jest.spyOn(Date.prototype, 'toISOString').mockReturnValue('2024-04-28T12:34:56.789Z'); - }); + describe('getWsseHeader', () => { + beforeEach(() => { + jest + .spyOn(crypto, 'randomBytes') + .mockReturnValue(Buffer.from('abcdef1234567890abcdef1234567890', 'hex')); + jest.spyOn(Date.prototype, 'toISOString').mockReturnValue('2024-04-28T12:34:56.789Z'); + }); - afterEach(() => { - jest.restoreAllMocks(); - }); + afterEach(() => { + jest.restoreAllMocks(); + }); - it('should generate a correct WSSE header', () => { - const user = 'testUser'; - const secret = 'testSecret'; - const expectedNonce = 'abcdef1234567890abcdef1234567890'; - const expectedTimestamp = '2024-04-28T12:34:56.789Z'; - const expectedDigest = base64Sha(expectedNonce + expectedTimestamp + secret); - const expectedHeader = `UsernameToken Username="${user}", PasswordDigest="${expectedDigest}", Nonce="${expectedNonce}", Created="${expectedTimestamp}"`; - const result = getWsseHeader(user, secret); + it('should generate a correct WSSE header', () => { + const user = 'testUser'; + const secret = 'testSecret'; + const expectedNonce = 'abcdef1234567890abcdef1234567890'; + const expectedTimestamp = '2024-04-28T12:34:56.789Z'; + const expectedDigest = base64Sha(expectedNonce + expectedTimestamp + secret); + const expectedHeader = `UsernameToken Username="${user}", PasswordDigest="${expectedDigest}", Nonce="${expectedNonce}", Created="${expectedTimestamp}"`; + const result = getWsseHeader(user, secret); - expect(result).toBe(expectedHeader); + expect(result).toBe(expectedHeader); + }); }); -}); -describe('buildIdentifyPayload', () => { - it('should correctly build payload with field mapping', () => { - const message = { - type: 'identify', - traits: { - firstName: 'John', - lastName: 'Doe', - email: 'john.doe@example.com', - optin: 1, - }, - }; - const destination = { - fieldMapping: [ - { rudderProperty: 'firstName', emersysProperty: '1' }, - { rudderProperty: 'lastName', emersysProperty: '2' }, - { rudderProperty: 'email', emersysProperty: '3' }, - { rudderProperty: 'optin', emersysProperty: '31' }, - ], - defaultContactList: 'dummyContactList', - }; - const expectedPayload = { - contact_list_id: 'dummyContactList', - contacts: [ - { - 1: 'John', - 2: 'Doe', - 3: 'john.doe@example.com', - 31: 1, + describe('buildIdentifyPayload', () => { + it('should correctly build payload with field mapping', () => { + const message = { + type: 'identify', + traits: { + firstName: 'John', + lastName: 'Doe', + email: 'john.doe@example.com', + optin: 1, }, - ], - key_id: 3, - }; + }; + const destination = { + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + defaultContactList: 'dummyContactList', + }; + const expectedPayload = { + contact_list_id: 'dummyContactList', + contacts: [ + { + 1: 'John', + 2: 'Doe', + 3: 'john.doe@example.com', + 31: 1, + }, + ], + key_id: 3, + }; - const result = buildIdentifyPayload(message, destination); + const result = buildIdentifyPayload(message, destination); - expect(result.eventType).toBe(EVENT_TYPE.IDENTIFY); - expect(result.destinationPayload).toEqual(expectedPayload); - }); + expect(result.eventType).toBe(EVENT_TYPE.IDENTIFY); + expect(result.destinationPayload).toEqual(expectedPayload); + }); - it('should throw error when opt-in field value is not allowed', () => { - const message = { - type: 'identify', - traits: { - firstName: 'John', - lastName: 'Doe', - email: 'john.doe@example.com', - optin: 3, - }, - }; - const destination = { - fieldMapping: [ - { rudderProperty: 'firstName', emersysProperty: '1' }, - { rudderProperty: 'lastName', emersysProperty: '2' }, - { rudderProperty: 'email', emersysProperty: '3' }, - { rudderProperty: 'optin', emersysProperty: '31' }, - ], - defaultContactList: 'dummyList', - }; - expect(() => { - buildIdentifyPayload(message, destination); - }).toThrow('Only 1,2, values are allowed for optin field'); - }); + it('should throw error when opt-in field value is not allowed', () => { + const message = { + type: 'identify', + traits: { + firstName: 'John', + lastName: 'Doe', + email: 'john.doe@example.com', + optin: 3, + }, + }; + const destination = { + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + defaultContactList: 'dummyList', + }; + expect(() => { + buildIdentifyPayload(message, destination); + }).toThrow('Only 1,2, values are allowed for optin field'); + }); - it('should throw error when no contact list can be assigned field value is not allowed', () => { - const message = { - type: 'identify', - traits: { - firstName: 'John', - lastName: 'Doe', - email: 'john.doe@example.com', - optin: 1, - }, - }; - const destination = { - fieldMapping: [ - { rudderProperty: 'firstName', emersysProperty: '1' }, - { rudderProperty: 'lastName', emersysProperty: '2' }, - { rudderProperty: 'email', emersysProperty: '3' }, - { rudderProperty: 'optin', emersysProperty: '31' }, - ], - }; - expect(() => { - buildIdentifyPayload(message, destination); - }).toThrow( - 'Cannot a find a specific contact list either through configuration or via integrations object', - ); - }); + it('should throw error when no contact list can be assigned field value is not allowed', () => { + const message = { + type: 'identify', + traits: { + firstName: 'John', + lastName: 'Doe', + email: 'john.doe@example.com', + optin: 1, + }, + }; + const destination = { + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + }; + expect(() => { + buildIdentifyPayload(message, destination); + }).toThrow( + 'Cannot a find a specific contact list either through configuration or via integrations object', + ); + }); - it('should correctly build payload with field mapping present in integrations object', () => { - const message = { - type: 'identify', - traits: { - firstName: 'John', - lastName: 'Doe', - email: 'john.doe@example.com', - optin: 1, - }, - integrations: { - EMARSYS: { - customIdentifierId: 1, - contactListId: 'objectListId', + it('should correctly build payload with field mapping present in integrations object', () => { + const message = { + type: 'identify', + traits: { + firstName: 'John', + lastName: 'Doe', + email: 'john.doe@example.com', + optin: 1, }, - }, - }; - const destination = { - fieldMapping: [ - { rudderProperty: 'firstName', emersysProperty: '1' }, - { rudderProperty: 'lastName', emersysProperty: '2' }, - { rudderProperty: 'email', emersysProperty: '3' }, - { rudderProperty: 'optin', emersysProperty: '31' }, - ], - defaultContactList: 'dummyContactList', - }; - const expectedPayload = { - contact_list_id: 'objectListId', - contacts: [ - { - 1: 'John', - 2: 'Doe', - 3: 'john.doe@example.com', - 31: 1, + integrations: { + EMARSYS: { + customIdentifierId: 1, + contactListId: 'objectListId', + }, }, - ], - key_id: 1, - }; + }; + const destination = { + fieldMapping: [ + { rudderProperty: 'firstName', emersysProperty: '1' }, + { rudderProperty: 'lastName', emersysProperty: '2' }, + { rudderProperty: 'email', emersysProperty: '3' }, + { rudderProperty: 'optin', emersysProperty: '31' }, + ], + defaultContactList: 'dummyContactList', + }; + const expectedPayload = { + contact_list_id: 'objectListId', + contacts: [ + { + 1: 'John', + 2: 'Doe', + 3: 'john.doe@example.com', + 31: 1, + }, + ], + key_id: 1, + }; - const result = buildIdentifyPayload(message, destination); + const result = buildIdentifyPayload(message, destination); - expect(result.eventType).toBe(EVENT_TYPE.IDENTIFY); - expect(result.destinationPayload).toEqual(expectedPayload); + expect(result.eventType).toBe(EVENT_TYPE.IDENTIFY); + expect(result.destinationPayload).toEqual(expectedPayload); + }); }); -}); -describe('buildGroupPayload', () => { - // Returns an object with eventType and destinationPayload keys when given valid message and destination inputs - it('should return an object with eventType and destinationPayload keys when given valid message and destination inputs with default externalId', () => { - const message = { - type: 'group', - groupId: 'group123', - context: { - traits: { - email: 'test@example.com', + describe('buildGroupPayload', () => { + // Returns an object with eventType and destinationPayload keys when given valid message and destination inputs + it('should return an object with eventType and destinationPayload keys when given valid message and destination inputs with default externalId', () => { + const message = { + type: 'group', + groupId: 'group123', + context: { + traits: { + email: 'test@example.com', + }, }, - }, - }; - const destination = { - Config: { - emersysCustomIdentifier: '3', + }; + const destination = { + Config: { + emersysCustomIdentifier: '3', + defaultContactList: 'list123', + fieldMapping: [ + { emersysProperty: '100', rudderProperty: 'customId' }, + { emersysProperty: '3', rudderProperty: 'email' }, + ], + }, + }; + const result = buildGroupPayload(message, destination); + expect(result).toEqual({ + eventType: 'group', + destinationPayload: { + payload: { + key_id: 3, + external_ids: ['test@example.com'], + }, + contactListId: 'group123', + }, + }); + }); + + it('should return an object with eventType and destinationPayload keys when given valid message and destination inputs with configured externalId', () => { + const message = { + type: 'group', + groupId: 'group123', + context: { + traits: { + email: 'test@example.com', + customId: '123', + }, + }, + }; + const destination = { + emersysCustomIdentifier: '100', defaultContactList: 'list123', fieldMapping: [ { emersysProperty: '100', rudderProperty: 'customId' }, { emersysProperty: '3', rudderProperty: 'email' }, ], - }, - }; - const result = buildGroupPayload(message, destination); - expect(result).toEqual({ - eventType: 'group', - destinationPayload: { - payload: { - key_id: 3, - external_ids: ['test@example.com'], + }; + const result = buildGroupPayload(message, destination); + expect(result).toEqual({ + eventType: 'group', + destinationPayload: { + payload: { + key_id: '100', + external_ids: ['123'], + }, + contactListId: 'group123', }, - contactListId: 'group123', - }, + }); }); - }); - it('should return an object with eventType and destinationPayload keys when given valid message and destination inputs with configured externalId', () => { - const message = { - type: 'group', - groupId: 'group123', - context: { - traits: { - email: 'test@example.com', - customId: '123', - }, - }, - }; - const destination = { - emersysCustomIdentifier: '100', - defaultContactList: 'list123', - fieldMapping: [ - { emersysProperty: '100', rudderProperty: 'customId' }, - { emersysProperty: '3', rudderProperty: 'email' }, - ], - }; - const result = buildGroupPayload(message, destination); - expect(result).toEqual({ - eventType: 'group', - destinationPayload: { - payload: { - key_id: '100', - external_ids: ['123'], + it('should throw an InstrumentationError if emersysCustomIdentifier value is not present in payload', () => { + const message = { + type: 'group', + groupId: 'group123', + context: { + traits: { + email: 'test@example.com', + }, }, - contactListId: 'group123', - }, + }; + const destination = { + emersysCustomIdentifier: 'customId', + defaultContactList: 'list123', + fieldMapping: [ + { emersysProperty: 'customId', rudderProperty: 'customId' }, + { emersysProperty: 'email', rudderProperty: 'email' }, + ], + }; + expect(() => { + buildGroupPayload(message, destination); + }).toThrow(InstrumentationError); }); }); - it('should throw an InstrumentationError if emersysCustomIdentifier value is not present in payload', () => { - const message = { - type: 'group', - groupId: 'group123', - context: { - traits: { - email: 'test@example.com', - }, - }, - }; - const destination = { - emersysCustomIdentifier: 'customId', - defaultContactList: 'list123', - fieldMapping: [ - { emersysProperty: 'customId', rudderProperty: 'customId' }, - { emersysProperty: 'email', rudderProperty: 'email' }, - ], - }; - expect(() => { - buildGroupPayload(message, destination); - }).toThrow(InstrumentationError); - }); -}); - -describe('createGroupBatches', () => { - // Should group events by key_id and contactListId - it('should group events by key_id and contactListId when events are provided', () => { - // Arrange - const events = [ - { - message: [ - { - body: { - JSON: { - destinationPayload: { - payload: { - key_id: 'key1', - external_ids: ['id1', 'id2'], + describe('createGroupBatches', () => { + // Should group events by key_id and contactListId + it('should group events by key_id and contactListId when events are provided', () => { + // Arrange + const events = [ + { + message: [ + { + body: { + JSON: { + destinationPayload: { + payload: { + key_id: 'key1', + external_ids: ['id1', 'id2'], + }, + contactListId: 'list1', }, - contactListId: 'list1', }, }, }, - }, - ], - metadata: { jobId: 1, userId: 'u1' }, - }, - { - message: [ - { - body: { - JSON: { - destinationPayload: { - payload: { - key_id: 'key2', - external_ids: ['id3', 'id4'], + ], + metadata: { jobId: 1, userId: 'u1' }, + }, + { + message: [ + { + body: { + JSON: { + destinationPayload: { + payload: { + key_id: 'key2', + external_ids: ['id3', 'id4'], + }, + contactListId: 'list2', }, - contactListId: 'list2', }, }, }, - }, - ], - metadata: { jobId: 2, userId: 'u2' }, - }, - { - message: [ - { - body: { - JSON: { - destinationPayload: { - payload: { - key_id: 'key1', - external_ids: ['id5', 'id6'], + ], + metadata: { jobId: 2, userId: 'u2' }, + }, + { + message: [ + { + body: { + JSON: { + destinationPayload: { + payload: { + key_id: 'key1', + external_ids: ['id5', 'id6'], + }, + contactListId: 'list1', }, - contactListId: 'list1', }, }, }, + ], + metadata: { jobId: 3, userId: 'u3' }, + }, + ]; + + // Act + const result = createGroupBatches(events); + + // Assert + expect(result).toEqual([ + { + endpoint: 'https://api.emarsys.net/api/v2/contactlist/list1/add', + payload: { + key_id: 'key1', + external_ids: ['id1', 'id2', 'id5', 'id6'], }, - ], - metadata: { jobId: 3, userId: 'u3' }, - }, - ]; - - // Act - const result = createGroupBatches(events); - - // Assert - expect(result).toEqual([ - { - endpoint: 'https://api.emarsys.net/api/v2/contactlist/list1/add', - payload: { - key_id: 'key1', - external_ids: ['id1', 'id2', 'id5', 'id6'], + metadata: [ + { jobId: 1, userId: 'u1' }, + { jobId: 3, userId: 'u3' }, + ], }, - metadata: [ - { jobId: 1, userId: 'u1' }, - { jobId: 3, userId: 'u3' }, - ], - }, - { - endpoint: 'https://api.emarsys.net/api/v2/contactlist/list2/add', - payload: { - key_id: 'key2', - external_ids: ['id3', 'id4'], + { + endpoint: 'https://api.emarsys.net/api/v2/contactlist/list2/add', + payload: { + key_id: 'key2', + external_ids: ['id3', 'id4'], + }, + metadata: [{ jobId: 2, userId: 'u2' }], }, - metadata: [{ jobId: 2, userId: 'u2' }], - }, - ]); - }); + ]); + }); - // Should return an empty array if no events are provided - it('should return an empty array when no events are provided', () => { - // Arrange - const events = []; + // Should return an empty array if no events are provided + it('should return an empty array when no events are provided', () => { + // Arrange + const events = []; - // Act - const result = createGroupBatches(events); + // Act + const result = createGroupBatches(events); - // Assert - expect(result).toEqual([]); + // Assert + expect(result).toEqual([]); + }); }); -}); -describe('findRudderPropertyByEmersysProperty', () => { - // Returns the correct rudderProperty when given a valid emersysProperty and fieldMapping - it('should return the correct rudderProperty when given a valid emersysProperty and fieldMapping', () => { - const emersysProperty = 'firstName'; - const fieldMapping = [ - { emersysProperty: 'email', rudderProperty: 'email' }, - { emersysProperty: 'firstName', rudderProperty: 'firstName' }, - { emersysProperty: 'lastName', rudderProperty: 'lastName' }, - ]; + describe('findRudderPropertyByEmersysProperty', () => { + // Returns the correct rudderProperty when given a valid emersysProperty and fieldMapping + it('should return the correct rudderProperty when given a valid emersysProperty and fieldMapping', () => { + const emersysProperty = 'firstName'; + const fieldMapping = [ + { emersysProperty: 'email', rudderProperty: 'email' }, + { emersysProperty: 'firstName', rudderProperty: 'firstName' }, + { emersysProperty: 'lastName', rudderProperty: 'lastName' }, + ]; - const result = findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping); + const result = findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping); - expect(result).toBe('firstName'); - }); + expect(result).toBe('firstName'); + }); - // Returns null when given an empty fieldMapping - it('should return null when given an empty fieldMapping', () => { - const emersysProperty = 'email'; - const fieldMapping = []; + // Returns null when given an empty fieldMapping + it('should return null when given an empty fieldMapping', () => { + const emersysProperty = 'email'; + const fieldMapping = []; - const result = findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping); + const result = findRudderPropertyByEmersysProperty(emersysProperty, fieldMapping); - expect(result).toBe('email'); + expect(result).toBe('email'); + }); }); -}); -describe('checkIfEventIsAbortableAndExtractErrorMessage', () => { - // Returns {isAbortable: false, errorMsg: ''} if event is neither a string nor an object with keyId. - it('should return {isAbortable: false, errorMsg: ""} when event is neither a string nor an object with keyId', () => { - const event = 123; - const destinationResponse = { - response: { - data: { - errors: { - errorKey: { - errorCode: 'errorMessage', + describe('checkIfEventIsAbortableAndExtractErrorMessage', () => { + // Returns {isAbortable: false, errorMsg: ''} if event is neither a string nor an object with keyId. + it('should return {isAbortable: false, errorMsg: ""} when event is neither a string nor an object with keyId', () => { + const event = 123; + const destinationResponse = { + response: { + data: { + errors: { + errorKey: { + errorCode: 'errorMessage', + }, }, }, }, - }, - }; - const keyId = 'keyId'; + }; + const keyId = 'keyId'; - const result = checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId); + const result = checkIfEventIsAbortableAndExtractErrorMessage( + event, + destinationResponse, + keyId, + ); - expect(result).toEqual({ isAbortable: false, errorMsg: '' }); - }); + expect(result).toEqual({ isAbortable: false, errorMsg: '' }); + }); - // Returns {isAbortable: false, errorMsg: ''} if errors object is empty. - it('should return {isAbortable: false, errorMsg: ""} when errors object is empty', () => { - const event = 'event'; - const destinationResponse = { - response: { - data: { - errors: {}, + // Returns {isAbortable: false, errorMsg: ''} if errors object is empty. + it('should return {isAbortable: false, errorMsg: ""} when errors object is empty', () => { + const event = 'event'; + const destinationResponse = { + response: { + data: { + errors: {}, + }, }, - }, - }; - const keyId = 'keyId'; + }; + const keyId = 'keyId'; - const result = checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId); + const result = checkIfEventIsAbortableAndExtractErrorMessage( + event, + destinationResponse, + keyId, + ); - expect(result).toEqual({ isAbortable: false, errorMsg: '' }); - }); + expect(result).toEqual({ isAbortable: false, errorMsg: '' }); + }); - // Returns {isAbortable: true, errorMsg} if event is a string and has a corresponding error in the errors object. - it('should return {isAbortable: true, errorMsg} when event is a string and has a corresponding error in the errors object', () => { - const event = 'event'; - const destinationResponse = { - response: { - data: { - errors: { - event: { - errorCode: 'errorMessage', + // Returns {isAbortable: true, errorMsg} if event is a string and has a corresponding error in the errors object. + it('should return {isAbortable: true, errorMsg} when event is a string and has a corresponding error in the errors object', () => { + const event = 'event'; + const destinationResponse = { + response: { + data: { + errors: { + event: { + errorCode: 'errorMessage', + }, }, }, }, - }, - }; - const keyId = 'keyId'; + }; + const keyId = 'keyId'; - const result = checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId); + const result = checkIfEventIsAbortableAndExtractErrorMessage( + event, + destinationResponse, + keyId, + ); - expect(result).toEqual({ isAbortable: true, errorMsg: '{"errorCode":"errorMessage"}' }); - }); + expect(result).toEqual({ isAbortable: true, errorMsg: '{"errorCode":"errorMessage"}' }); + }); - // Returns {isAbortable: true, errorMsg} if event is an object with keyId and has a corresponding error in the errors object. - it('should return {isAbortable: true, errorMsg} when event is an object with keyId and has a corresponding error in the errors object', () => { - const event = { - keyId: 'event', - }; - const destinationResponse = { - response: { - data: { - errors: { - event: { - errorCode: 'errorMessage', + // Returns {isAbortable: true, errorMsg} if event is an object with keyId and has a corresponding error in the errors object. + it('should return {isAbortable: true, errorMsg} when event is an object with keyId and has a corresponding error in the errors object', () => { + const event = { + keyId: 'event', + }; + const destinationResponse = { + response: { + data: { + errors: { + event: { + errorCode: 'errorMessage', + }, }, }, }, - }, - }; - const keyId = 'keyId'; + }; + const keyId = 'keyId'; + + const result = checkIfEventIsAbortableAndExtractErrorMessage( + event, + destinationResponse, + keyId, + ); + + expect(result).toEqual({ isAbortable: true, errorMsg: '{"errorCode":"errorMessage"}' }); + }); + }); + + describe('deduceEventId', () => { + // When a valid event name is provided and there is a mapping for it, the function should return the corresponding eventId. + it('should return the corresponding eventId when a valid event name is provided and there is a mapping for it', () => { + const message = { event: 'validEvent' }; + const destConfig = { eventsMapping: [{ from: 'validEvent', to: 'eventId' }] }; + const result = deduceEventId(message, destConfig); + expect(result).toBe('eventId'); + }); - const result = checkIfEventIsAbortableAndExtractErrorMessage(event, destinationResponse, keyId); + // When an invalid event name is provided, the function should throw a ConfigurationError. + it('should throw a ConfigurationError when an invalid event name is provided', () => { + const message = { event: 'invalidEvent' }; + const destConfig = { eventsMapping: [{ from: 'validEvent', to: 'eventId' }] }; + expect(() => deduceEventId(message, destConfig)).toThrow(ConfigurationError); + }); - expect(result).toEqual({ isAbortable: true, errorMsg: '{"errorCode":"errorMessage"}' }); + // When a valid event name is provided and there is no mapping for it, the function should throw a ConfigurationError. + it('should throw a ConfigurationError when a valid event name is provided and there is no mapping for it', () => { + const message = { event: 'validEvent' }; + const destConfig = { eventsMapping: [] }; + expect(() => deduceEventId(message, destConfig)).toThrow(ConfigurationError); + }); + + // When eventsMapping is not an array, the function should throw a TypeError. + it('should throw a TypeError when eventsMapping is not an array', () => { + const message = { event: 'validEvent' }; + const destConfig = { eventsMapping: 'notAnArray' }; + expect(() => deduceEventId(message, destConfig)).toThrow( + 'validEvent is not mapped to any Emersys external event. Aborting', + ); + }); }); }); diff --git a/src/v1/destinations/emarsys/networkHandler.js b/src/v1/destinations/emarsys/networkHandler.js index c88d2b02dd..cef8013028 100644 --- a/src/v1/destinations/emarsys/networkHandler.js +++ b/src/v1/destinations/emarsys/networkHandler.js @@ -98,19 +98,18 @@ const responseHandler = (responseParams) => { }; } - // otherwise all events are successful - responseWithIndividualEvents = rudderJobMetadata.map((metadata) => ({ - statusCode: 200, - metadata, - error: 'success', - })); + // ref : https://dev.emarsys.com/docs/emarsys-core-api-guides/45c776d275862-http-500-errors - return { + throw new TransformerProxyError( + `EMARSYS: Error transformer proxy v1 during EMARSYS response transformation`, status, - message, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, destinationResponse, - response: responseWithIndividualEvents, - }; + '', + responseWithIndividualEvents, + ); }; function networkHandler() {