Skip to content

Commit

Permalink
fix: identify batching final
Browse files Browse the repository at this point in the history
  • Loading branch information
shrouti1507 committed May 2, 2024
1 parent 608469b commit 68ab695
Show file tree
Hide file tree
Showing 7 changed files with 315 additions and 264 deletions.
2 changes: 1 addition & 1 deletion src/cdk/v2/destinations/emarsys/procWorkflow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ steps:
ref: https://dev.emarsys.com/docs/core-api-reference/fl0xx6rwfbwqb-trigger-an-external-event
condition: $.outputs.messageType === {{$.EventType.TRACK}}
template: |
const integrationObject = $.getIntegrationsObj(message, 'emersys');
const integrationObject = $.getIntegrationsObj(^.message, 'emersys');
const emersysIdentifierId = integrationObject?.customIdentifierId || ^.destination.Config.emersysCustomIdentifier || $.EMAIL_FIELD_ID;
const payload = {
key_id: emersysIdentifierId ,
Expand Down
6 changes: 3 additions & 3 deletions src/cdk/v2/destinations/emarsys/rtWorkflow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ steps:
- name: batchSuccessfulEvents
description: Batches the successfulEvents
debug: true
template: |
$.batchResponseBuilder($.outputs.successfulEvents);
$.context.batchedPayload = $.batchResponseBuilder($.outputs.successfulEvents);
console.log("batchedPayload",JSON.stringify($.context.batchedPayload));
- name: finalPayload
template: |
[...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents]
[...$.outputs.failedEvents, ...$.context.batchedPayload]
33 changes: 17 additions & 16 deletions src/cdk/v2/destinations/emarsys/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@ const buildHeader = (destConfig) => {
};
};

const buildIdentifyPayload = (message, destination) => {
const buildIdentifyPayload = (message, destConfig) => {
let destinationPayload;
const { fieldMapping, emersysCustomIdentifier, discardEmptyProperties, defaultContactList } =
destination.Config;
destConfig;
const payload = {};

const integrationObject = getIntegrationsObj(message, 'emersys');
const integrationObject = getIntegrationsObj(message, 'emarsys');
const finalContactList = integrationObject?.contactListId || defaultContactList;

if (!isDefinedAndNotNullAndNotEmpty(finalContactList)) {
Expand All @@ -65,9 +65,10 @@ const buildIdentifyPayload = (message, destination) => {
if (fieldMapping) {
fieldMapping.forEach((trait) => {
const { rudderProperty, emersysProperty } = trait;
const value =
getValueFromMessage(message.traits, rudderProperty) ||
getValueFromMessage(message.context.traits, rudderProperty);
const value = getValueFromMessage(message, [
`traits.${rudderProperty}`,
`context.traits.${rudderProperty}`,
]);
if (value) {
payload[emersysProperty] = value;
}
Expand Down Expand Up @@ -99,7 +100,6 @@ const buildIdentifyPayload = (message, destination) => {
'Either configured custom contact identifier value or default identifier email value is missing',
);
}

return { eventType: message.type, destinationPayload };
};

Expand Down Expand Up @@ -248,17 +248,16 @@ const createIdentifyBatches = (events) => {
const groupedIdentifyPayload = lodash.groupBy(
events,
(item) =>
`${item.message.body.JSON.destinationPayload.key_id}-${item.message.body.JSON.destinationPayload.contact_list_id}`,
`${item.message[0].body.JSON.destinationPayload.key_id}-${item.message[0].body.JSON.destinationPayload.contact_list_id}`,
);

return lodash.flatMap(groupedIdentifyPayload, (group) => {
const firstItem = group[0].message.body.JSON.destinationPayload;
const firstItem = group[0].message[0].body.JSON.destinationPayload;
// eslint-disable-next-line @typescript-eslint/naming-convention
const { key_id, contact_list_id } = firstItem;

const allContacts = lodash.flatMap(
group,
(item) => item.message.body.JSON.destinationPayload.contacts,
(item) => item.message[0].body.JSON.destinationPayload.contacts,
);
const initialChunks = lodash.chunk(allContacts, MAX_BATCH_SIZE);
const finalChunks = lodash.flatMap(initialChunks, ensureSizeConstraints);
Expand Down Expand Up @@ -306,9 +305,10 @@ const createTrackBatches = (events) => ({
metadata: events[0].metadata,
});
const formatIdentifyPayloadsWithEndpoint = (combinedPayloads, endpointUrl = '') =>
combinedPayloads.map((payload) => ({
combinedPayloads.map((singleCombinedPayload) => ({
endpoint: endpointUrl,
payload,
payload: singleCombinedPayload.payload,
metadata: singleCombinedPayload.metadata,
}));

const buildBatchedRequest = (batches, method, constants, batchedStatus = true) =>
Expand Down Expand Up @@ -349,12 +349,12 @@ const batchResponseBuilder = (successfulEvents) => {

const typedEventGroups = lodash.groupBy(
successfulEvents,
(event) => event.message.body.JSON.eventType,
(event) => event.message[0].body.JSON.eventType,
);
Object.keys(typedEventGroups).forEach((eachEventGroup) => {
switch (eachEventGroup) {
case EventType.IDENTIFY:
batchesOfIdentifyEvents = createIdentifyBatches(eachEventGroup);
batchesOfIdentifyEvents = createIdentifyBatches(typedEventGroups[eachEventGroup]);
groupedSuccessfulPayload.identify.batches = formatIdentifyPayloadsWithEndpoint(
batchesOfIdentifyEvents,
'https://api.emarsys.net/api/v2/contact/?create_if_not_exists=1',
Expand All @@ -371,6 +371,7 @@ const batchResponseBuilder = (successfulEvents) => {
}
return groupedSuccessfulPayload;
});
console.log('groupedSuccessfulPayload', JSON.stringify(groupedSuccessfulPayload));

Check warning on line 374 in src/cdk/v2/destinations/emarsys/utils.js

View workflow job for this annotation

GitHub Actions / Check for formatting & lint errors

Unexpected console statement
// Process each identify batch
if (groupedSuccessfulPayload.identify) {
const identifyBatches = buildBatchedRequest(
Expand Down Expand Up @@ -400,7 +401,7 @@ const batchResponseBuilder = (successfulEvents) => {
);
finaloutput.push(...trackBatches);
}

console.log('FINAL', JSON.stringify(finaloutput));

Check warning on line 404 in src/cdk/v2/destinations/emarsys/utils.js

View workflow job for this annotation

GitHub Actions / Check for formatting & lint errors

Unexpected console statement
return finaloutput;
};

Expand Down
64 changes: 28 additions & 36 deletions src/cdk/v2/destinations/emarsys/utils.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,13 @@ describe('buildIdentifyPayload', () => {
},
};
const destination = {
Config: {
fieldMapping: [
{ rudderProperty: 'firstName', emersysProperty: '1' },
{ rudderProperty: 'lastName', emersysProperty: '2' },
{ rudderProperty: 'email', emersysProperty: '3' },
{ rudderProperty: 'optin', emersysProperty: '31' },
],
defaultContactList: 'dummyContactList',
},
fieldMapping: [
{ rudderProperty: 'firstName', emersysProperty: '1' },
{ rudderProperty: 'lastName', emersysProperty: '2' },
{ rudderProperty: 'email', emersysProperty: '3' },
{ rudderProperty: 'optin', emersysProperty: '31' },
],
defaultContactList: 'dummyContactList',
};
const expectedPayload = {
contact_list_id: 'dummyContactList',
Expand Down Expand Up @@ -106,15 +104,13 @@ describe('buildIdentifyPayload', () => {
},
};
const destination = {
Config: {
fieldMapping: [
{ rudderProperty: 'firstName', emersysProperty: '1' },
{ rudderProperty: 'lastName', emersysProperty: '2' },
{ rudderProperty: 'email', emersysProperty: '3' },
{ rudderProperty: 'optin', emersysProperty: '31' },
],
defaultContactList: 'dummyList',
},
fieldMapping: [
{ rudderProperty: 'firstName', emersysProperty: '1' },
{ rudderProperty: 'lastName', emersysProperty: '2' },
{ rudderProperty: 'email', emersysProperty: '3' },
{ rudderProperty: 'optin', emersysProperty: '31' },
],
defaultContactList: 'dummyList',
};
expect(() => {
buildIdentifyPayload(message, destination);
Expand All @@ -132,14 +128,12 @@ describe('buildIdentifyPayload', () => {
},
};
const destination = {
Config: {
fieldMapping: [
{ rudderProperty: 'firstName', emersysProperty: '1' },
{ rudderProperty: 'lastName', emersysProperty: '2' },
{ rudderProperty: 'email', emersysProperty: '3' },
{ rudderProperty: 'optin', emersysProperty: '31' },
],
},
fieldMapping: [
{ rudderProperty: 'firstName', emersysProperty: '1' },
{ rudderProperty: 'lastName', emersysProperty: '2' },
{ rudderProperty: 'email', emersysProperty: '3' },
{ rudderProperty: 'optin', emersysProperty: '31' },
],
};
expect(() => {
buildIdentifyPayload(message, destination);
Expand All @@ -158,22 +152,20 @@ describe('buildIdentifyPayload', () => {
optin: 1,
},
integrations: {
EMERSYS: {
EMARSYS: {
customIdentifierId: 1,
contactListId: 'objectListId',
},
},
};
const destination = {
Config: {
fieldMapping: [
{ rudderProperty: 'firstName', emersysProperty: '1' },
{ rudderProperty: 'lastName', emersysProperty: '2' },
{ rudderProperty: 'email', emersysProperty: '3' },
{ rudderProperty: 'optin', emersysProperty: '31' },
],
defaultContactList: 'dummyContactList',
},
fieldMapping: [
{ rudderProperty: 'firstName', emersysProperty: '1' },
{ rudderProperty: 'lastName', emersysProperty: '2' },
{ rudderProperty: 'email', emersysProperty: '3' },
{ rudderProperty: 'optin', emersysProperty: '31' },
],
defaultContactList: 'dummyContactList',
};
const expectedPayload = {
contact_list_id: 'objectListId',
Expand Down
2 changes: 1 addition & 1 deletion src/constants/destinationCanonicalNames.js
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ const DestCanonicalNames = {
],
koala: ['Koala', 'koala', 'KOALA'],
bloomreach: ['Bloomreach', 'bloomreach', 'BLOOMREACH'],
emersys: ['EMARSYS', 'Emarsys', 'emarsys'],
emarsys: ['EMARSYS', 'Emarsys', 'emarsys'],
};

module.exports = { DestHandlerMap, DestCanonicalNames };
82 changes: 82 additions & 0 deletions src/v0/destinations/emarsys/deleteUsers.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
const { NetworkError, ConfigurationError } = require('@rudderstack/integrations-lib');
const { httpPOST } = require('../../../adapters/network');
const {
processAxiosResponse,
getDynamicErrorType,
} = require('../../../adapters/utils/networkUtils');
const { isHttpStatusSuccess } = require('../../util');
const { executeCommonValidations } = require('../../util/regulation-api');
const tags = require('../../util/tags');
const { getUserIdBatches } = require('../../util/deleteUserUtils');
const { JSON_MIME_TYPE } = require('../../util/constant');

/**
* This function will help to delete the users one by one from the userAttributes array.
* @param {*} userAttributes Array of objects with userId, email and phone
* @param {*} config Destination.Config provided in dashboard
* @returns
*/
const userDeletionHandler = async (userAttributes, config) => {
const { apiKey } = config;

if (!apiKey) {
throw new ConfigurationError('Api Key is required for user deletion');
}

const endpoint = 'https://api.sprig.com/v2/purge/visitors';
const headers = {
Accept: JSON_MIME_TYPE,
'Content-Type': JSON_MIME_TYPE,
Authorization: `API-Key ${apiKey}`,
};
/**
* userIdBatches = [[u1,u2,u3,...batchSize],[u1,u2,u3,...batchSize]...]
* Ref doc : https://docs.sprig.com/reference/post-v2-purge-visitors-1
*/
const userIdBatches = getUserIdBatches(userAttributes, 100);
// Note: we will only get 400 status code when no user deletion is present for given userIds so we will not throw error in that case
// eslint-disable-next-line no-restricted-syntax
for (const curBatch of userIdBatches) {
// eslint-disable-next-line no-await-in-loop
const deletionResponse = await httpPOST(
endpoint,
{
userIds: curBatch,
},
{
headers,
},
{
destType: 'sprig',
feature: 'deleteUsers',
endpointPath: '/purge/visitors',
requestMethod: 'POST',
module: 'deletion',
},
);
const handledDelResponse = processAxiosResponse(deletionResponse);
if (!isHttpStatusSuccess(handledDelResponse.status) && handledDelResponse.status !== 400) {
throw new NetworkError(
'User deletion request failed',
handledDelResponse.status,
{
[tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(handledDelResponse.status),
[tags.TAG_NAMES.STATUS]: handledDelResponse.status,
},
handledDelResponse,
);
}
}

return {
statusCode: 200,
status: 'successful',
};
};
const processDeleteUsers = async (event) => {
const { userAttributes, config } = event;
executeCommonValidations(userAttributes);
const resp = await userDeletionHandler(userAttributes, config);
return resp;
};
module.exports = { processDeleteUsers };
Loading

0 comments on commit 68ab695

Please sign in to comment.