From 2f5fcb01d1d55d9a6d37cf100d34522a1ca53616 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Mon, 15 Jul 2024 09:22:00 +0530 Subject: [PATCH 01/26] chore: small fixes --- src/v0/destinations/klaviyo/config.js | 35 ++- .../klaviyo/data/KlaviyoProfileV2.json | 128 +++++++++ .../klaviyo/data/KlaviyoTrackV2.json | 31 +++ src/v0/destinations/klaviyo/transform.js | 10 +- src/v0/destinations/klaviyo/transformV2.js | 223 ++++++++++++++++ src/v0/destinations/klaviyo/util.js | 250 +++++++++++++++++- src/v0/destinations/klaviyo/util.test.js | 70 +++++ .../destinations/klaviyo/processor/data.ts | 2 + .../destinations/klaviyo/processor/dataV2.ts | 13 + .../klaviyo/processor/screenTestDataV2.ts | 148 +++++++++++ .../klaviyo/processor/validationTestDataV2.ts | 83 ++++++ 11 files changed, 978 insertions(+), 15 deletions(-) create mode 100644 src/v0/destinations/klaviyo/data/KlaviyoProfileV2.json create mode 100644 src/v0/destinations/klaviyo/data/KlaviyoTrackV2.json create mode 100644 src/v0/destinations/klaviyo/transformV2.js create mode 100644 src/v0/destinations/klaviyo/util.test.js create mode 100644 test/integrations/destinations/klaviyo/processor/dataV2.ts create mode 100644 test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts create mode 100644 test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts diff --git a/src/v0/destinations/klaviyo/config.js b/src/v0/destinations/klaviyo/config.js index d8583ab9cb..ed17d42c9a 100644 --- a/src/v0/destinations/klaviyo/config.js +++ b/src/v0/destinations/klaviyo/config.js @@ -9,10 +9,12 @@ const MAX_BATCH_SIZE = 100; const CONFIG_CATEGORIES = { IDENTIFY: { name: 'KlaviyoIdentify', apiUrl: '/api/profiles' }, - SCREEN: { name: 'KlaviyoTrack', apiUrl: '/api/events' }, + IDENTIFYV2: { name: 'KlaviyoProfileV2', apiUrl: '/api/profile-import' }, TRACK: { name: 'KlaviyoTrack', apiUrl: '/api/events' }, + TRACKV2: { name: 'KlaviyoTrackV2', apiUrl: '/api/events' }, GROUP: { name: 'KlaviyoGroup' }, PROFILE: { name: 'KlaviyoProfile' }, + PROFILEV2: { name: 'KlaviyoProfileV2' }, STARTED_CHECKOUT: { name: 'StartedCheckout' }, VIEWED_PRODUCT: { name: 'ViewedProduct' }, ADDED_TO_CART: { name: 'AddedToCart' }, @@ -57,8 +59,35 @@ const LIST_CONF = { }; const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); -const destType = 'klaviyo'; +const WhiteListedTraitsV2 = [ + 'email', + 'firstName', + 'firstname', + 'first_name', + 'lastName', + 'lastname', + 'last_name', + 'phone', + 'title', + 'organization', + 'city', + 'region', + 'country', + 'zip', + 'image', + 'timezone', + 'anonymousId', + 'userId', + 'properties', + 'location', + '_kx', + 'street', + 'address', +]; +const destType = 'klaviyo'; +// api version used +const revision = '2024-06-15'; module.exports = { BASE_ENDPOINT, MAX_BATCH_SIZE, @@ -70,4 +99,6 @@ module.exports = { eventNameMapping, jsonNameMapping, destType, + revision, + WhiteListedTraitsV2, }; diff --git a/src/v0/destinations/klaviyo/data/KlaviyoProfileV2.json b/src/v0/destinations/klaviyo/data/KlaviyoProfileV2.json new file mode 100644 index 0000000000..310f3eba38 --- /dev/null +++ b/src/v0/destinations/klaviyo/data/KlaviyoProfileV2.json @@ -0,0 +1,128 @@ +[ + { + "destKey": "external_id", + "sourceKeys": "userIdOnly", + "sourceFromGenericMap": true + }, + { + "destKey": "anonymous_id", + "sourceKeys": "anonymousId" + }, + { + "destKey": "email", + "sourceKeys": "emailOnly", + "sourceFromGenericMap": true + }, + { + "destKey": "first_name", + "sourceKeys": "firstName", + "sourceFromGenericMap": true + }, + { + "destKey": "last_name", + "sourceKeys": "lastName", + "sourceFromGenericMap": true + }, + { + "destKey": "phone_number", + "sourceKeys": "phone", + "sourceFromGenericMap": true + }, + { + "destKey": "title", + "sourceKeys": ["traits.title", "context.traits.title", "properties.title"] + }, + { + "destKey": "organization", + "sourceKeys": ["traits.organization", "context.traits.organization", "properties.organization"] + }, + { + "destKey": "location.city", + "sourceKeys": [ + "traits.city", + "traits.address.city", + "context.traits.city", + "context.traits.address.city", + "properties.city" + ] + }, + { + "destKey": "location.region", + "sourceKeys": [ + "traits.region", + "traits.address.region", + "context.traits.region", + "context.traits.address.region", + "properties.region", + "traits.state", + "traits.address.state", + "context.traits.address.state", + "context.traits.state", + "properties.state" + ] + }, + { + "destKey": "location.country", + "sourceKeys": [ + "traits.country", + "traits.address.country", + "context.traits.country", + "context.traits.address.country", + "properties.country" + ] + }, + { + "destKey": "location.zip", + "sourceKeys": [ + "traits.zip", + "traits.postalcode", + "traits.postalCode", + "traits.address.zip", + "traits.address.postalcode", + "traits.address.postalCode", + "context.traits.zip", + "context.traits.postalcode", + "context.traits.postalCode", + "context.traits.address.zip", + "context.traits.address.postalcode", + "context.traits.address.postalCode", + "properties.zip", + "properties.postalcode", + "properties.postalCode" + ] + }, + { + "destKey": "location.ip", + "sourceKeys": ["context.ip", "request_ip"] + }, + { + "destKey": "_kx", + "sourceKeys": ["traits._kx", "context.traits._kx"] + }, + { + "destKey": "image", + "sourceKeys": ["traits.image", "context.traits.image", "properties.image"] + }, + { + "destKey": "location.timezone", + "sourceKeys": ["traits.timezone", "context.traits.timezone", "properties.timezone"] + }, + { + "destKey": "location.latitude", + "sourceKeys": ["latitude", "context.address.latitude", "context.location.latitude"] + }, + { + "destKey": "location.longitude", + "sourceKeys": ["longitude", "context.address.longitude", "context.location.longitude"] + }, + { + "destKey": "location.address1", + "sourceKeys": [ + "traits.street", + "traits.address.street", + "context.traits.street", + "context.traits.address.street", + "properties.street" + ] + } +] diff --git a/src/v0/destinations/klaviyo/data/KlaviyoTrackV2.json b/src/v0/destinations/klaviyo/data/KlaviyoTrackV2.json new file mode 100644 index 0000000000..8fbf1f4191 --- /dev/null +++ b/src/v0/destinations/klaviyo/data/KlaviyoTrackV2.json @@ -0,0 +1,31 @@ +[ + { + "destKey": "value", + "sourceKeys": ["properties.revenue", "properties.total", "properties.value"] + }, + { + "destKey": "value_currency", + "sourceKeys": "properties.currency" + }, + { + "destKey": "time", + "sourceKeys": "timestamp", + "sourceFromGenericMap": true + }, + { + "destKey": "properties", + "sourceKeys": "properties", + "metadata": { + "excludes": [ + "event", + "email", + "phone", + "revenue", + "total", + "value", + "value_currency", + "currency" + ] + } + } +] diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js index 09e75919f9..3877ebb61a 100644 --- a/src/v0/destinations/klaviyo/transform.js +++ b/src/v0/destinations/klaviyo/transform.js @@ -13,6 +13,7 @@ const { eventNameMapping, jsonNameMapping, } = require('./config'); +const { processRouterDestV2, processV2 } = require('./transformV2'); const { createCustomerProperties, subscribeUserToList, @@ -277,6 +278,9 @@ const groupRequestHandler = (message, category, destination) => { // Main event processor using specific handler funcs const processEvent = async (event, reqMetadata) => { const { message, destination, metadata } = event; + if (destination.Config?.version === 'v2' || message.version === 'v2') { + return processV2(event, reqMetadata); + } if (!message.type) { throw new InstrumentationError('Event type is required'); } @@ -327,11 +331,15 @@ const getEventChunks = (event, subscribeRespList, nonSubscribeRespList) => { }; const processRouterDest = async (inputs, reqMetadata) => { + const { destination } = inputs[0]; + // This is used to switch to latest API version + if (destination.Config?.version === 'v2') { + return processRouterDestV2(inputs, reqMetadata); + } let batchResponseList = []; const batchErrorRespList = []; const subscribeRespList = []; const nonSubscribeRespList = []; - const { destination } = inputs[0]; await Promise.all( inputs.map(async (event) => { try { diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js new file mode 100644 index 0000000000..46e175e1e0 --- /dev/null +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -0,0 +1,223 @@ +/* eslint-disable no-nested-ternary */ +/* eslint-disable no-underscore-dangle */ +/* eslint-disable array-callback-return */ +const get = require('get-value'); +const { ConfigurationError, InstrumentationError } = require('@rudderstack/integrations-lib'); +const { EventType } = require('../../../constants'); +const { CONFIG_CATEGORIES, BASE_ENDPOINT, MAPPING_CONFIG, revision } = require('./config'); +const { batchSubscribeEvents, constructProfile, subscribeUserToListV2 } = require('./util'); +const { + defaultRequestConfig, + constructPayload, + getFieldValueFromMessage, + defaultPostRequestConfig, + removeUndefinedAndNullValues, + getSuccessRespEvents, + handleRtTfSingleEventError, + flattenJson, +} = require('../../util'); +const { JSON_MIME_TYPE } = require('../../util/constant'); + +/** + * Main Identify request handler func + * The function is used to create/update new users and also for subscribing + * users to the list. + * DOCS: 1. https://developers.klaviyo.com/en/reference/create_or_update_profile + * 2. https://developers.klaviyo.com/en/reference/subscribe_profiles + * @param {*} message + * @param {*} category + * @param {*} destination + * @returns + */ +const identifyRequestHandler = async (message, category, destination) => { + // If listId property is present try to subscribe/member user in list + const { privateApiKey, listId } = destination.Config; + const payload = removeUndefinedAndNullValues(constructProfile(message, destination, true)); + const endpoint = `${BASE_ENDPOINT}${category.apiUrl}`; + const requestOptions = { + headers: { + Authorization: `Klaviyo-API-Key ${privateApiKey}`, + accept: JSON_MIME_TYPE, + 'content-type': JSON_MIME_TYPE, + revision, + }, + }; + const profileRequest = defaultRequestConfig(); + profileRequest.endpoint = endpoint; + profileRequest.body.JSON = payload; + profileRequest.headers = requestOptions.headers; + let responseList = profileRequest; + const traitsInfo = getFieldValueFromMessage(message, 'traits'); + // check if user wants to subscribe profile or not and listId is present or not + if (traitsInfo?.properties?.subscribe && (traitsInfo.properties?.listId || listId)) { + responseList = [responseList, subscribeUserToListV2(message, traitsInfo, destination)]; + } + // returning list if subscription to a list is to be done else returning an object to upsert profile + return responseList; +}; + +/** + * Main handler func for track/screen request + * User info needs to be mapped to a track event (mandatory) + * DOCS: https://developers.klaviyo.com/en/reference/create_event + * @param {*} message + * @param {*} category + * @param {*} destination + * @returns requestBody + */ +const trackOrScreenRequestHandler = (message, category, destination) => { + const payload = {}; + const { privateApiKey, flattenProperties } = destination.Config; + // event for track and name for screen call + const event = get(message, 'event') || get(message, 'name'); + if (event && typeof event !== 'string') { + throw new InstrumentationError('Event type should be a string'); + } + const attributes = constructPayload(message, MAPPING_CONFIG[category.name]); + + // if flattenProperties is enabled from UI, flatten the event properties + attributes.properties = flattenProperties + ? flattenJson(attributes.properties, '.', 'normal', false) + : attributes.properties; + + // get profile object + attributes.profile = removeUndefinedAndNullValues(constructProfile(message, destination, false)); + attributes.metric = { + data: { + type: 'metric', + attributes: { + name: event, + }, + }, + }; + payload.data = { type: 'event', attributes }; + const response = defaultRequestConfig(); + response.endpoint = `${BASE_ENDPOINT}${category.apiUrl}`; + response.method = defaultPostRequestConfig.requestMethod; + response.headers = { + Authorization: `Klaviyo-API-Key ${privateApiKey}`, + 'Content-Type': JSON_MIME_TYPE, + Accept: JSON_MIME_TYPE, + revision, + }; + response.body.JSON = removeUndefinedAndNullValues(payload); + return response; +}; + +/** + * Main handlerfunc for group request add/subscribe users to the list based on property sent + * DOCS:https://developers.klaviyo.com/en/reference/subscribe_profiles + * @param {*} message + * @param {*} category + * @param {*} destination + * @returns + */ +const groupRequestHandler = (message, category, destination) => { + if (!message.groupId) { + throw new InstrumentationError('groupId is a required field for group events'); + } + const traitsInfo = getFieldValueFromMessage(message, 'traits'); + if (!traitsInfo?.subscribe) { + throw new InstrumentationError('Subscribe flag should be true for group call'); + } + + return [subscribeUserToListV2(message, traitsInfo, destination)]; +}; + +const processV2 = async (event, reqMetadata) => { + const { message, destination, metadata } = event; + if (!message.type) { + throw new InstrumentationError('Event type is required'); + } + if (!destination.Config.privateApiKey) { + throw new ConfigurationError(`Private API Key is a required field for ${message.type} events`); + } + const messageType = message.type.toLowerCase(); + + let category; + let response; + switch (messageType) { + case EventType.IDENTIFY: + category = CONFIG_CATEGORIES.IDENTIFYV2; + response = await identifyRequestHandler( + { message, category, destination, metadata }, + reqMetadata, + ); + break; + case EventType.SCREEN: + case EventType.TRACK: + category = CONFIG_CATEGORIES.TRACKV2; + response = trackOrScreenRequestHandler(message, category, destination); + break; + case EventType.GROUP: + category = CONFIG_CATEGORIES.GROUP; + response = groupRequestHandler(message, category, destination); + break; + default: + throw new InstrumentationError(`Event type ${messageType} is not supported`); + } + return response; +}; + +// This function separates subscribe response and other responses in chunks +const getEventChunks = (event, subscribeRespList, nonSubscribeRespList) => { + if (Array.isArray(event.message)) { + // this list contains responses for subscribe endpoint + subscribeRespList.push(event); + } else { + // this list doesn't contain subsribe endpoint responses + nonSubscribeRespList.push(event); + } +}; + +const processRouterDestV2 = async (inputs, reqMetadata) => { + let batchResponseList = []; + const batchErrorRespList = []; + const subscribeRespList = []; + const nonSubscribeRespList = []; + const { destination } = inputs[0]; + await Promise.all( + inputs.map(async (event) => { + try { + if (event.message.statusCode) { + // already transformed event + getEventChunks( + { message: event.message, metadata: event.metadata, destination }, + subscribeRespList, + nonSubscribeRespList, + ); + } else { + // if not transformed + getEventChunks( + { + message: await process(event, reqMetadata), + metadata: event.metadata, + destination, + }, + subscribeRespList, + nonSubscribeRespList, + ); + } + } catch (error) { + const errRespEvent = handleRtTfSingleEventError(event, error, reqMetadata); + batchErrorRespList.push(errRespEvent); + } + }), + ); + const batchedSubscribeResponseList = []; + if (subscribeRespList.length > 0) { + const batchedResponseList = batchSubscribeEvents(subscribeRespList, 'v2'); + batchedSubscribeResponseList.push(...batchedResponseList); + } + const nonSubscribeSuccessList = nonSubscribeRespList.map((resp) => { + const response = resp; + const { message, metadata, destination: eventDestination } = response; + return getSuccessRespEvents(message, [metadata], eventDestination); + }); + + batchResponseList = [...batchedSubscribeResponseList, ...nonSubscribeSuccessList]; + + return [...batchResponseList, ...batchErrorRespList]; +}; + +module.exports = { processV2, processRouterDestV2 }; diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 3f6f6ca2ca..fe554f4a51 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -2,7 +2,6 @@ const { defaultRequestConfig } = require('rudder-transformer-cdk/build/utils'); const lodash = require('lodash'); const { NetworkError, InstrumentationError } = require('@rudderstack/integrations-lib'); const { WhiteListedTraits } = require('../../../constants'); - const { constructPayload, getFieldValueFromMessage, @@ -11,13 +10,24 @@ const { removeUndefinedAndNullValues, defaultBatchRequestConfig, getSuccessRespEvents, + flattenJson, defaultPatchRequestConfig, + isDefinedAndNotNull, + getDestinationExternalID, + getIntegrationsObj, } = require('../../util'); const tags = require('../../util/tags'); const { handleHttpRequest } = require('../../../adapters/network'); const { JSON_MIME_TYPE, HTTP_STATUS_CODES } = require('../../util/constant'); const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); -const { BASE_ENDPOINT, MAPPING_CONFIG, CONFIG_CATEGORIES, MAX_BATCH_SIZE } = require('./config'); +const { + BASE_ENDPOINT, + MAPPING_CONFIG, + CONFIG_CATEGORIES, + MAX_BATCH_SIZE, + WhiteListedTraitsV2, + revision, +} = require('./config'); const REVISION_CONSTANT = '2023-02-22'; @@ -203,7 +213,7 @@ const populateCustomFieldsFromTraits = (message) => { return customProperties; }; -const generateBatchedPaylaodForArray = (events) => { +const generateBatchedPaylaodForArray = (events, version) => { let batchEventResponse = defaultBatchRequestConfig(); const batchResponseList = []; const metadata = []; @@ -213,6 +223,10 @@ const generateBatchedPaylaodForArray = (events) => { events.forEach((ev, index) => { if (index === 0) { batchResponseList.push(ev.message.body.JSON); + } else if (version === 'v2') { + batchResponseList[0].data.attributes.profiles.data.push( + ...ev.message.body.JSON.data.attributes.profiles.data, + ); } else { batchResponseList[0].data.attributes.subscriptions.push( ...ev.message.body.JSON.data.attributes.subscriptions, @@ -248,23 +262,25 @@ const generateBatchedPaylaodForArray = (events) => { /** * It takes list of subscribe responses and groups them on the basis of listId * @param {*} subscribeResponseList + * @param {*} version this parameter to know the API and accordingly fetch listId from payload from right place * @returns */ -const groupSubscribeResponsesUsingListId = (subscribeResponseList) => { - const subscribeEventGroups = lodash.groupBy( - subscribeResponseList, - (event) => event.message.body.JSON.data.attributes.list_id, +const groupSubscribeResponsesUsingListId = (subscribeResponseList, version) => { + const subscribeEventGroups = lodash.groupBy(subscribeResponseList, (event) => + version === 'v2' + ? event.message.body.JSON.data.relationships.list.data.id + : event.message.body.JSON.data.attributes.list_id, ); return subscribeEventGroups; }; -const getBatchedResponseList = (subscribeEventGroups, identifyResponseList) => { +const getBatchedResponseList = (subscribeEventGroups, identifyResponseList, version) => { let batchedResponseList = []; Object.keys(subscribeEventGroups).forEach((listId) => { // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] const eventChunks = lodash.chunk(subscribeEventGroups[listId], MAX_BATCH_SIZE); const batchedResponse = eventChunks.map((chunk) => { - const batchEventResponse = generateBatchedPaylaodForArray(chunk); + const batchEventResponse = generateBatchedPaylaodForArray(chunk, version); return getSuccessRespEvents( batchEventResponse.batchedRequest, batchEventResponse.metadata, @@ -284,7 +300,7 @@ const getBatchedResponseList = (subscribeEventGroups, identifyResponseList) => { return batchedResponseList; }; -const batchSubscribeEvents = (subscribeRespList) => { +const batchSubscribeEvents = (subscribeRespList, version = 'v1') => { const identifyResponseList = []; subscribeRespList.forEach((event) => { const processedEvent = event; @@ -299,13 +315,220 @@ const batchSubscribeEvents = (subscribeRespList) => { } }); - const subscribeEventGroups = groupSubscribeResponsesUsingListId(subscribeRespList); + const subscribeEventGroups = groupSubscribeResponsesUsingListId(subscribeRespList, version); - const batchedResponseList = getBatchedResponseList(subscribeEventGroups, identifyResponseList); + const batchedResponseList = getBatchedResponseList( + subscribeEventGroups, + identifyResponseList, + version, + ); return batchedResponseList; }; +/** + * This function generates the metadat object used for updating a list attribute and unset properties + * message = { + integrations: { + Klaviyo: { fieldsToUnset: ['Unset1', 'Unset2'], + fieldsToAppend: ['appendList1', 'appendList2'], + fieldsToAppend: ['unappendList1', 'unappendList2'] + }, + All: true, + }, + }; + Output metadata = { + "meta": { + "patch_properties": { + "append": { + "appendList1": "New Value 1", + "appendList2": "New Value 2" + }, + "unappend": { + "unappendList1": "Old Value 1", + "unappendList2": "Old Value 2" + }, + "unset": ['Unset1', 'Unset2'] + } + } + } + * @param {*} message + */ +const getProfileMetadataAndMetadataFields = (message) => { + const intgObj = getIntegrationsObj(message, 'Klaviyo'); + const meta = { patch_properties: {} }; + let metadataFields = []; + const traitsInfo = getFieldValueFromMessage(message, 'traits'); + // fetch and set fields to unset + const fieldsToUnset = intgObj?.fieldsToUnset; + if (Array.isArray(fieldsToUnset)) { + meta.patch_properties.unset = fieldsToUnset; + metadataFields = fieldsToUnset; + } + + // fetch list of fields to append , their value and append these fields in metadataFields + const fieldsToAppend = intgObj?.fieldsToAppend; + if (Array.isArray(fieldsToAppend)) { + const append = {}; + fieldsToAppend.forEach((key) => { + if (isDefinedAndNotNull(traitsInfo[key])) { + append[key] = traitsInfo[key]; + } + }); + meta.patch_properties.append = append; + metadataFields = metadataFields.concat(fieldsToAppend); + } + + // fetch list of fields to unappend , their value and append these fields in metadataFields + const fieldsToUnappend = intgObj?.fieldsToUnappend; + if (Array.isArray(fieldsToUnappend)) { + const unappend = {}; + fieldsToUnappend.forEach((key) => { + if (isDefinedAndNotNull(traitsInfo[key])) { + unappend[key] = traitsInfo[key]; + } + }); + meta.patch_properties.unappend = unappend; + metadataFields = metadataFields.concat(fieldsToUnappend); + } + + return { meta, metadataFields }; +}; + +/** + * Following function is used to construct profile object for version 15-06-2024 + * If we have isIdentifyCall as true then there are two extra scenarios + * 1. `enforceEmailAsPrimary` config kicks in and if email or phone is not present we throw an error + * 2. Place of Metadata object in payload for klaviyo is a little but different + * @param {*} message input to build output from + * @param {*} destination dest object with config + * @param {*} isIdentifyCall let us know if processing is done for identify call + * @returns profile object + * https://developers.klaviyo.com/en/reference/create_or_update_profile + */ +const constructProfile = (message, destination, isIdentifyCall) => { + const profileAttributes = constructPayload( + message, + MAPPING_CONFIG[CONFIG_CATEGORIES.PROFILEV2.name], + ); + const { enforceEmailAsPrimary, flattenProperties } = destination.Config; + let customPropertyPayload = {}; + const { meta, metadataFields } = getProfileMetadataAndMetadataFields(message); + customPropertyPayload = extractCustomFields( + message, + customPropertyPayload, + ['traits', 'context.traits'], + // omitting whitelistedTraitsV2 and metadatafields from constructing custom properties as these are already used + [...WhiteListedTraitsV2, ...metadataFields], + ); + const profileId = getDestinationExternalID(message, 'klaviyo-profileId'); + // if flattenProperties is enabled from UI, flatten the user properties + customPropertyPayload = flattenProperties + ? flattenJson(customPropertyPayload, '.', 'normal', false) + : customPropertyPayload; + if (isIdentifyCall && enforceEmailAsPrimary) { + if (!profileAttributes.email && !profileAttributes.phone_number && !profileId) { + throw new InstrumentationError('None of email and phone are present in the payload'); + } + delete profileAttributes.external_id; // so that multiple profiles are not found, one w.r.t email and one for external_id + customPropertyPayload = { + ...customPropertyPayload, + _id: getFieldValueFromMessage(message, 'userIdOnly'), // custom attribute + }; + } + const data = removeUndefinedAndNullValues({ + type: 'profile', + id: profileId, + attributes: { + ...profileAttributes, + properties: removeUndefinedAndNullValues(customPropertyPayload), + meta, + }, + }); + if (isIdentifyCall) { + // For identify call meta object comes inside + data.metadata = meta; + delete data.attributes.meta; + } + + return { data }; +}; + +/** + * This function is used for creating response for subscribing users to a particular list for V2 + * DOCS: https://developers.klaviyo.com/en/reference/subscribe_profiles + */ +const subscribeUserToListV2 = (message, traitsInfo, destination) => { + // listId from message properties are preferred over Config listId + const { privateApiKey, consent } = destination.Config; + let { listId } = destination.Config; + let subscribeConsent = traitsInfo?.properties?.consent || consent; + const subscriptions = {}; + const email = getFieldValueFromMessage(message, 'email'); + const phone = getFieldValueFromMessage(message, 'phone'); + if (subscribeConsent) { + if (!Array.isArray(subscribeConsent)) { + subscribeConsent = [subscribeConsent]; + } + if (subscribeConsent.includes('email') && email) { + subscriptions.email.marketing.consent = 'SUBSCRIBED'; + } + if (subscribeConsent.includes('sms') && phone) { + subscriptions.sms.marketing.consent = 'SUBSCRIBED'; + } + } + const profileAttributes = { + email, + phone_number: phone, + subscriptions, + }; + const profile = removeUndefinedAndNullValues({ + type: 'profile', + id: getDestinationExternalID(message, 'klaviyo-profileId'), + attributes: removeUndefinedAndNullValues(profileAttributes), + }); + if (!email && !phone && profile.id) { + throw new InstrumentationError( + 'Profile Id, Email or/and Phone are required to subscribe to a list', + ); + } + // fetch list id from message + if (traitsInfo?.properties?.listId) { + // for identify call + listId = traitsInfo.properties.listId; + } + if (message.type === 'group') { + listId = message.groupId; + } + + const payload = { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { profiles: { data: [profile] } }, + relationships: { + list: { + data: { + type: 'list', + id: listId, + }, + }, + }, + }, + }; + const response = defaultRequestConfig(); + response.method = defaultPostRequestConfig.requestMethod; + response.endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; + response.headers = { + Authorization: `Klaviyo-API-Key ${privateApiKey}`, + 'Content-Type': JSON_MIME_TYPE, + Accept: JSON_MIME_TYPE, + revision, + }; + response.body.JSON = removeUndefinedAndNullValues(payload); + + return response; +}; + module.exports = { subscribeUserToList, createCustomerProperties, @@ -314,4 +537,7 @@ module.exports = { batchSubscribeEvents, profileUpdateResponseBuilder, getIdFromNewOrExistingProfile, + constructProfile, + subscribeUserToListV2, + getProfileMetadataAndMetadataFields, }; diff --git a/src/v0/destinations/klaviyo/util.test.js b/src/v0/destinations/klaviyo/util.test.js new file mode 100644 index 0000000000..36749e2311 --- /dev/null +++ b/src/v0/destinations/klaviyo/util.test.js @@ -0,0 +1,70 @@ +const { getProfileMetadataAndMetadataFields } = require('./util'); + +describe('getProfileMetadataAndMetadataFields', () => { + // Correctly generates metadata with fields to unset, append, and unappend when all fields are provided + it('should generate metadata with fields to unset, append, and unappend when all fields are provided', () => { + const message = { + integrations: { + Klaviyo: { + fieldsToUnset: ['Unset1', 'Unset2'], + fieldsToAppend: ['appendList1', 'appendList2'], + fieldsToUnappend: ['unappendList1', 'unappendList2'], + }, + All: true, + }, + traits: { + appendList1: 'New Value 1', + appendList2: 'New Value 2', + unappendList1: 'Old Value 1', + unappendList2: 'Old Value 2', + }, + }; + + jest.mock('../../util', () => ({ + getIntegrationsObj: jest.fn(() => message.integrations.Klaviyo), + getFieldValueFromMessage: jest.fn(() => message.traits), + isDefinedAndNotNull: jest.fn((value) => value !== null && value !== undefined), + })); + + const result = getProfileMetadataAndMetadataFields(message); + + expect(result).toEqual({ + meta: { + patch_properties: { + append: { + appendList1: 'New Value 1', + appendList2: 'New Value 2', + }, + unappend: { + unappendList1: 'Old Value 1', + unappendList2: 'Old Value 2', + }, + unset: ['Unset1', 'Unset2'], + }, + }, + metadataFields: [ + 'Unset1', + 'Unset2', + 'appendList1', + 'appendList2', + 'unappendList1', + 'unappendList2', + ], + }); + }); + + // Handles null or undefined message input gracefully + it('should return empty metadata and metadataFields when message is null or undefined', () => { + jest.mock('../../util', () => ({ + getIntegrationsObj: jest.fn(() => null), + getFieldValueFromMessage: jest.fn(() => ({})), + isDefinedAndNotNull: jest.fn((value) => value !== null && value !== undefined), + })); + + let result = getProfileMetadataAndMetadataFields(null); + expect(result).toEqual({ meta: { patch_properties: {} }, metadataFields: [] }); + + result = getProfileMetadataAndMetadataFields(undefined); + expect(result).toEqual({ meta: { patch_properties: {} }, metadataFields: [] }); + }); +}); diff --git a/test/integrations/destinations/klaviyo/processor/data.ts b/test/integrations/destinations/klaviyo/processor/data.ts index 06c4a3e530..71a10fc39c 100644 --- a/test/integrations/destinations/klaviyo/processor/data.ts +++ b/test/integrations/destinations/klaviyo/processor/data.ts @@ -4,8 +4,10 @@ import { identifyData } from './identifyTestData'; import { screenTestData } from './screenTestData'; import { trackTestData } from './trackTestData'; import { validationTestData } from './validationTestData'; +import { dataV2 } from './dataV2'; export const data = [ + ...dataV2, ...identifyData, ...trackTestData, ...screenTestData, diff --git a/test/integrations/destinations/klaviyo/processor/dataV2.ts b/test/integrations/destinations/klaviyo/processor/dataV2.ts new file mode 100644 index 0000000000..6fb55c0736 --- /dev/null +++ b/test/integrations/destinations/klaviyo/processor/dataV2.ts @@ -0,0 +1,13 @@ +// import { groupTestData } from './groupTestDataV2'; +// import { identifyData } from './identifyTestDataV2'; +import { screenTestData } from './screenTestDataV2'; +// import { trackTestData } from './trackTestDataV2'; +import { validationTestData } from './validationTestData'; + +export const dataV2 = [ + // ...identifyData, + // ...trackTestData, + ...screenTestData, + // ...groupTestData, + ...validationTestData, +]; diff --git a/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts new file mode 100644 index 0000000000..41caa1d1bb --- /dev/null +++ b/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts @@ -0,0 +1,148 @@ +import { Destination } from '../../../../../src/types'; +import { ProcessorTestData } from '../../../testTypes'; +import { + generateMetadata, + generateSimplifiedPageOrScreenPayload, + transformResultBuilder, +} from '../../../testUtils'; + +const destination: Destination = { + ID: '123', + Name: 'klaviyo', + DestinationDefinition: { + ID: '123', + Name: 'klaviyo', + DisplayName: 'klaviyo', + Config: {}, + }, + Config: { + version: 'v2', + privateApiKey: 'dummyPrivateApiKey', + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; + +export const screenTestData: ProcessorTestData[] = [ + { + id: 'klaviyo-screen-test-1', + name: 'klaviyo', + description: 'Screen event call with properties and contextual traits', + scenario: 'Business', + successCriteria: + 'Response should contain only event payload and status code should be 200, for the event payload should contain properties and contextual traits in the payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: generateSimplifiedPageOrScreenPayload( + { + name: 'Viewed Screen', + sentAt: '2021-01-25T16:12:02.048Z', + userId: 'sajal12', + integrations: { + klaviyo: { + fieldsToAppend: ['append1'], + }, + }, + context: { + traits: { + id: 'user@1', + age: '22', + email: 'test@rudderstack.com', + phone: '9112340375', + anonymousId: '9c6bd77ea9da3e68', + append1: 'value1', + }, + }, + properties: { + value: 9.99, + currency: 'USD', + PreviouslyVicePresident: true, + YearElected: 1801, + VicePresidents: ['Aaron Burr', 'George Clinton'], + }, + anonymousId: '9c6bd77ea9da3e68', + originalTimestamp: '2021-01-25T15:32:56.409Z', + }, + 'screen', + ), + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/events', + headers: { + Accept: 'application/json', + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + JSON: { + data: { + type: 'event', + attributes: { + time: '2021-01-25T15:32:56.409Z', + value: 9.99, + value_currency: 'USD', + metric: { + data: { + type: 'metric', + attributes: { + name: 'Viewed Screen', + }, + }, + }, + properties: { + PreviouslyVicePresident: true, + YearElected: 1801, + VicePresidents: ['Aaron Burr', 'George Clinton'], + }, + profile: { + data: { + attributes: { + anonymous_id: '9c6bd77ea9da3e68', + external_id: 'sajal12', + email: 'test@rudderstack.com', + meta: { + patch_properties: { + append: { + append1: 'value1', + }, + }, + }, + phone_number: '9112340375', + properties: { + id: 'user@1', + age: '22', + }, + }, + type: 'profile', + }, + }, + }, + }, + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts new file mode 100644 index 0000000000..9d673e8fc7 --- /dev/null +++ b/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts @@ -0,0 +1,83 @@ +import { Destination } from '../../../../../src/types'; +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata } from '../../../testUtils'; + +const destination: Destination = { + ID: '123', + Name: 'klaviyo', + DestinationDefinition: { + ID: '123', + Name: 'klaviyo', + DisplayName: 'klaviyo', + Config: {}, + }, + Config: { + privateApiKey: 'dummyPrivateApiKey', + version: 'v2', + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; + +export const validationTestData: ProcessorTestData[] = [ + { + id: 'klaviyo-validation-test-1', + name: 'klaviyo', + description: '[Error]: Check for unsupported message type', + scenario: 'Framework', + successCriteria: + 'Response should contain error message and status code should be 400, as we are sending a message type which is not supported by Klaviyo destination and the error message should be Event type random is not supported', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + userId: 'user123', + type: 'random', + groupId: 'XUepkK', + traits: { + subscribe: true, + }, + context: { + traits: { + email: 'test@rudderstack.com', + phone: '+12 345 678 900', + consent: 'email', + }, + }, + timestamp: '2020-01-21T00:21:34.208Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: 'Event type random is not supported', + statTags: { + destType: 'KLAVIYO', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + }, + statusCode: 400, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; From 4c402fc76483c9617295342af88fb77af135861b Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Mon, 15 Jul 2024 09:31:35 +0530 Subject: [PATCH 02/26] chore: small fixes+1 --- src/constants/destinationCanonicalNames.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/constants/destinationCanonicalNames.js b/src/constants/destinationCanonicalNames.js index 915ac50b26..c529a91a48 100644 --- a/src/constants/destinationCanonicalNames.js +++ b/src/constants/destinationCanonicalNames.js @@ -173,6 +173,7 @@ const DestCanonicalNames = { 'Klaviyo Bulk Upload', 'klaviyobulkupload', ], + Klaviyo: ['KLAVIYO', 'Klaviyo', 'klaviyo'], emarsys: ['EMARSYS', 'Emarsys', 'emarsys'], wunderkind: ['wunderkind', 'Wunderkind', 'WUNDERKIND'], }; From 4459b99be671b867e3165236322bc1284cdfcff1 Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Mon, 15 Jul 2024 10:22:27 +0530 Subject: [PATCH 03/26] chore: tmp update transform.js for version switching --- src/v0/destinations/klaviyo/transform.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js index 3877ebb61a..79ea7e9153 100644 --- a/src/v0/destinations/klaviyo/transform.js +++ b/src/v0/destinations/klaviyo/transform.js @@ -278,7 +278,7 @@ const groupRequestHandler = (message, category, destination) => { // Main event processor using specific handler funcs const processEvent = async (event, reqMetadata) => { const { message, destination, metadata } = event; - if (destination.Config?.version === 'v2' || message.version === 'v2') { + if (destination.Config?.version === 'v2' || message.context.version === 'v2') { return processV2(event, reqMetadata); } if (!message.type) { @@ -333,7 +333,7 @@ const getEventChunks = (event, subscribeRespList, nonSubscribeRespList) => { const processRouterDest = async (inputs, reqMetadata) => { const { destination } = inputs[0]; // This is used to switch to latest API version - if (destination.Config?.version === 'v2') { + if (destination.Config?.version === 'v2' || message.context.version === 'v2') { return processRouterDestV2(inputs, reqMetadata); } let batchResponseList = []; From b3b0a32c74c07143c1b34cb6fffdbb9f8a91790b Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Mon, 15 Jul 2024 10:30:13 +0530 Subject: [PATCH 04/26] Update transform.js --- src/v0/destinations/klaviyo/transform.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js index 79ea7e9153..a6511812e6 100644 --- a/src/v0/destinations/klaviyo/transform.js +++ b/src/v0/destinations/klaviyo/transform.js @@ -331,9 +331,9 @@ const getEventChunks = (event, subscribeRespList, nonSubscribeRespList) => { }; const processRouterDest = async (inputs, reqMetadata) => { - const { destination } = inputs[0]; + const { destination, event } = inputs[0]; // This is used to switch to latest API version - if (destination.Config?.version === 'v2' || message.context.version === 'v2') { + if (destination.Config?.version === 'v2' || event.message.context.version === 'v2') { return processRouterDestV2(inputs, reqMetadata); } let batchResponseList = []; From 928aa815f8a64aa1016a3a7c0c6def7bbf348484 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Mon, 15 Jul 2024 10:45:50 +0530 Subject: [PATCH 05/26] chore: tmp commit --- src/v0/destinations/klaviyo/transform.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js index a6511812e6..88fcdb5ae4 100644 --- a/src/v0/destinations/klaviyo/transform.js +++ b/src/v0/destinations/klaviyo/transform.js @@ -331,9 +331,9 @@ const getEventChunks = (event, subscribeRespList, nonSubscribeRespList) => { }; const processRouterDest = async (inputs, reqMetadata) => { - const { destination, event } = inputs[0]; + const { destination } = inputs[0]; // This is used to switch to latest API version - if (destination.Config?.version === 'v2' || event.message.context.version === 'v2') { + if (destination.Config?.version === 'v2' || inputs[0]?.message?.context.version === 'v2') { return processRouterDestV2(inputs, reqMetadata); } let batchResponseList = []; From c523a10fbfcd9338f8cb9edf5557757790cc6c6c Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Mon, 15 Jul 2024 11:55:41 +0530 Subject: [PATCH 06/26] chore: tmp commit --- src/v0/destinations/klaviyo/config.js | 3 +- src/v0/destinations/klaviyo/transform.js | 5 +- src/v0/destinations/klaviyo/transformV2.js | 17 +- src/v0/destinations/klaviyo/util.js | 22 +- .../destinations/klaviyo/processor/dataV2.ts | 4 +- .../klaviyo/processor/identifyTestDataV2.ts | 294 ++++++++++++++++++ 6 files changed, 319 insertions(+), 26 deletions(-) create mode 100644 test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts diff --git a/src/v0/destinations/klaviyo/config.js b/src/v0/destinations/klaviyo/config.js index ed17d42c9a..54216852f7 100644 --- a/src/v0/destinations/klaviyo/config.js +++ b/src/v0/destinations/klaviyo/config.js @@ -57,7 +57,7 @@ const LIST_CONF = { SUBSCRIBE: 'subscribe_with_consentInfo', ADD_TO_LIST: 'subscribe_without_consentInfo', }; - +const useUpdatedKlaviyoAPI = process.env.USE_UPDATED_KLAVIYO_API === 'true' || false; const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); const WhiteListedTraitsV2 = [ @@ -101,4 +101,5 @@ module.exports = { destType, revision, WhiteListedTraitsV2, + useUpdatedKlaviyoAPI, }; diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js index 88fcdb5ae4..a63fa65bb8 100644 --- a/src/v0/destinations/klaviyo/transform.js +++ b/src/v0/destinations/klaviyo/transform.js @@ -12,6 +12,7 @@ const { ecomEvents, eventNameMapping, jsonNameMapping, + useUpdatedKlaviyoAPI, } = require('./config'); const { processRouterDestV2, processV2 } = require('./transformV2'); const { @@ -278,7 +279,7 @@ const groupRequestHandler = (message, category, destination) => { // Main event processor using specific handler funcs const processEvent = async (event, reqMetadata) => { const { message, destination, metadata } = event; - if (destination.Config?.version === 'v2' || message.context.version === 'v2') { + if (destination.Config?.version === 'v2' || useUpdatedKlaviyoAPI) { return processV2(event, reqMetadata); } if (!message.type) { @@ -333,7 +334,7 @@ const getEventChunks = (event, subscribeRespList, nonSubscribeRespList) => { const processRouterDest = async (inputs, reqMetadata) => { const { destination } = inputs[0]; // This is used to switch to latest API version - if (destination.Config?.version === 'v2' || inputs[0]?.message?.context.version === 'v2') { + if (destination.Config?.version === 'v2' || useUpdatedKlaviyoAPI) { return processRouterDestV2(inputs, reqMetadata); } let batchResponseList = []; diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index 46e175e1e0..5e2cf7f19f 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -29,7 +29,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); * @param {*} destination * @returns */ -const identifyRequestHandler = async (message, category, destination) => { +const identifyRequestHandler = (message, category, destination) => { // If listId property is present try to subscribe/member user in list const { privateApiKey, listId } = destination.Config; const payload = removeUndefinedAndNullValues(constructProfile(message, destination, true)); @@ -37,8 +37,8 @@ const identifyRequestHandler = async (message, category, destination) => { const requestOptions = { headers: { Authorization: `Klaviyo-API-Key ${privateApiKey}`, - accept: JSON_MIME_TYPE, - 'content-type': JSON_MIME_TYPE, + Accept: JSON_MIME_TYPE, + 'Content-Type': JSON_MIME_TYPE, revision, }, }; @@ -96,8 +96,8 @@ const trackOrScreenRequestHandler = (message, category, destination) => { response.method = defaultPostRequestConfig.requestMethod; response.headers = { Authorization: `Klaviyo-API-Key ${privateApiKey}`, - 'Content-Type': JSON_MIME_TYPE, Accept: JSON_MIME_TYPE, + 'Content-Type': JSON_MIME_TYPE, revision, }; response.body.JSON = removeUndefinedAndNullValues(payload); @@ -124,8 +124,8 @@ const groupRequestHandler = (message, category, destination) => { return [subscribeUserToListV2(message, traitsInfo, destination)]; }; -const processV2 = async (event, reqMetadata) => { - const { message, destination, metadata } = event; +const processV2 = async (event) => { + const { message, destination } = event; if (!message.type) { throw new InstrumentationError('Event type is required'); } @@ -139,10 +139,7 @@ const processV2 = async (event, reqMetadata) => { switch (messageType) { case EventType.IDENTIFY: category = CONFIG_CATEGORIES.IDENTIFYV2; - response = await identifyRequestHandler( - { message, category, destination, metadata }, - reqMetadata, - ); + response = identifyRequestHandler(message, category, destination); break; case EventType.SCREEN: case EventType.TRACK: diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index fe554f4a51..bd6c3b2456 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -1,3 +1,4 @@ +const set = require('set-value'); const { defaultRequestConfig } = require('rudder-transformer-cdk/build/utils'); const lodash = require('lodash'); const { NetworkError, InstrumentationError } = require('@rudderstack/integrations-lib'); @@ -332,7 +333,7 @@ const batchSubscribeEvents = (subscribeRespList, version = 'v1') => { integrations: { Klaviyo: { fieldsToUnset: ['Unset1', 'Unset2'], fieldsToAppend: ['appendList1', 'appendList2'], - fieldsToAppend: ['unappendList1', 'unappendList2'] + fieldsToUnappend: ['unappendList1', 'unappendList2'] }, All: true, }, @@ -447,7 +448,7 @@ const constructProfile = (message, destination, isIdentifyCall) => { }); if (isIdentifyCall) { // For identify call meta object comes inside - data.metadata = meta; + data.meta = meta; delete data.attributes.meta; } @@ -463,25 +464,24 @@ const subscribeUserToListV2 = (message, traitsInfo, destination) => { const { privateApiKey, consent } = destination.Config; let { listId } = destination.Config; let subscribeConsent = traitsInfo?.properties?.consent || consent; - const subscriptions = {}; const email = getFieldValueFromMessage(message, 'email'); const phone = getFieldValueFromMessage(message, 'phone'); + const profileAttributes = { + email, + phone_number: phone, + }; if (subscribeConsent) { if (!Array.isArray(subscribeConsent)) { subscribeConsent = [subscribeConsent]; } if (subscribeConsent.includes('email') && email) { - subscriptions.email.marketing.consent = 'SUBSCRIBED'; + set(profileAttributes, 'subscriptions.email.marketing.consent', 'SUBSCRIBED'); } if (subscribeConsent.includes('sms') && phone) { - subscriptions.sms.marketing.consent = 'SUBSCRIBED'; + set(profileAttributes, 'subscriptions.sms.marketing.consent', 'SUBSCRIBED'); } } - const profileAttributes = { - email, - phone_number: phone, - subscriptions, - }; + const profile = removeUndefinedAndNullValues({ type: 'profile', id: getDestinationExternalID(message, 'klaviyo-profileId'), @@ -520,8 +520,8 @@ const subscribeUserToListV2 = (message, traitsInfo, destination) => { response.endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; response.headers = { Authorization: `Klaviyo-API-Key ${privateApiKey}`, - 'Content-Type': JSON_MIME_TYPE, Accept: JSON_MIME_TYPE, + 'Content-Type': JSON_MIME_TYPE, revision, }; response.body.JSON = removeUndefinedAndNullValues(payload); diff --git a/test/integrations/destinations/klaviyo/processor/dataV2.ts b/test/integrations/destinations/klaviyo/processor/dataV2.ts index 6fb55c0736..118a41164e 100644 --- a/test/integrations/destinations/klaviyo/processor/dataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/dataV2.ts @@ -1,11 +1,11 @@ // import { groupTestData } from './groupTestDataV2'; -// import { identifyData } from './identifyTestDataV2'; +import { identifyData } from './identifyTestDataV2'; import { screenTestData } from './screenTestDataV2'; // import { trackTestData } from './trackTestDataV2'; import { validationTestData } from './validationTestData'; export const dataV2 = [ - // ...identifyData, + ...identifyData, // ...trackTestData, ...screenTestData, // ...groupTestData, diff --git a/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts new file mode 100644 index 0000000000..65356e9031 --- /dev/null +++ b/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts @@ -0,0 +1,294 @@ +import { removeUndefinedAndNullValues } from '@rudderstack/integrations-lib'; +import { + overrideDestination, + transformResultBuilder, + generateSimplifiedIdentifyPayload, + generateMetadata, +} from '../../../testUtils'; +import { ProcessorTestData } from '../../../testTypes'; +import { Destination } from '../../../../../src/types'; + +const destination: Destination = { + ID: '123', + Name: 'klaviyo', + DestinationDefinition: { + ID: '123', + Name: 'klaviyo', + DisplayName: 'klaviyo', + Config: {}, + }, + Config: { + version: 'v2', + privateApiKey: 'dummyPrivateApiKey', + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; + +const commonTraits = { + firstName: 'Test', + lastName: 'Rudderlabs', + email: 'test@rudderstack.com', + phone: '+12 345 578 900', + userId: 'user@1', + title: 'Developer', + organization: 'Rudder', + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + Flagged: false, + Residence: 'Shibuya', + street: '63, Shibuya', + properties: { + listId: 'XUepkK', + subscribe: true, + consent: ['email', 'sms'], + }, +}; + +const commonOutputUserProps = { + external_id: 'user@1', + anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + email: 'test@rudderstack.com', + first_name: 'Test', + last_name: 'Rudderlabs', + phone_number: '+12 345 578 900', + title: 'Developer', + organization: 'Rudder', + location: { + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + address1: '63, Shibuya', + }, + properties: { + Flagged: false, + Residence: 'Shibuya', + }, +}; + +const commonOutputSubscriptionProps = { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'test@rudderstack.com', + phone_number: '+12 345 578 900', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + sms: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + ], + }, +}; +const subscriptionRelations = { + list: { + data: { + type: 'list', + id: 'XUepkK', + }, + }, +}; + +const commonOutputHeaders = { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + 'Content-Type': 'application/json', + Accept: 'application/json', + revision: '2024-06-15', +}; + +const anonymousId = '97c46c81-3140-456d-b2a9-690d70aaca35'; +const userId = 'user@1'; +const sentAt = '2021-01-03T17:02:53.195Z'; +const originalTimestamp = '2021-01-03T17:02:53.193Z'; +const userProfileCommonEndpoint = 'https://a.klaviyo.com/api/profile-import'; +const subscribeEndpoint = 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs'; + +export const identifyData: ProcessorTestData[] = [ + { + id: 'klaviyo-identify-150624-test-1', + name: 'klaviyo', + description: + '150624 -> Identify call with flattenProperties enabled in destination config and subscribe', + scenario: 'Business', + successCriteria: + 'The profile response should contain the flattened properties of the friend object and one request object for subscribe', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { flattenProperties: true }), + message: generateSimplifiedIdentifyPayload({ + sentAt, + userId, + context: { + traits: { + ...commonTraits, + friend: { + names: { + first: 'Alice', + last: 'Smith', + }, + age: 25, + }, + }, + }, + anonymousId, + originalTimestamp, + }), + metadata: generateMetadata(2), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers: commonOutputHeaders, + JSON: { + data: { + type: 'profile', + attributes: { + ...commonOutputUserProps, + properties: { + ...commonOutputUserProps.properties, + 'friend.age': 25, + 'friend.names.first': 'Alice', + 'friend.names.last': 'Smith', + }, + }, + meta: { + patch_properties: {}, + }, + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(2), + }, + { + output: transformResultBuilder({ + userId: '', + method: 'POST', + endpoint: subscribeEndpoint, + headers: commonOutputHeaders, + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: commonOutputSubscriptionProps, + relationships: subscriptionRelations, + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(2), + }, + ], + }, + }, + }, + { + id: 'klaviyo-identify-150624-test-2', + name: 'klaviyo', + description: + '150624 -> Profile without subscribing the user and get klaviyo id from externalId', + scenario: 'Business', + successCriteria: + 'Response should contain only profile update payload and status code should be 200 as subscribe is set to false in the payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: generateSimplifiedIdentifyPayload({ + sentAt, + userId, + context: { + externalId: [ + { + type: 'klaviyo-profileId', + id: '12345678', + }, + ], + traits: { + ...commonTraits, + appendList1: 'New Value 1', + appendList2: 'New Value 2', + unappendList1: 'Old Value 1', + unappendList2: 'Old Value 2', + properties: { ...commonTraits.properties, subscribe: false }, + }, + }, + integrations: { + Klaviyo: { + fieldsToUnset: ['Unset1', 'Unset2'], + fieldsToAppend: ['appendList1', 'appendList2'], + fieldsToUnappend: ['unappendList1', 'unappendList2'], + }, + All: true, + }, + anonymousId, + originalTimestamp, + }), + metadata: generateMetadata(4), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers: commonOutputHeaders, + JSON: { + data: { + type: 'profile', + id: '12345678', + attributes: commonOutputUserProps, + meta: { + patch_properties: { + append: { + appendList1: 'New Value 1', + appendList2: 'New Value 2', + }, + unappend: { + unappendList1: 'Old Value 1', + unappendList2: 'Old Value 2', + }, + unset: ['Unset1', 'Unset2'], + }, + }, + }, + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(4), + }, + ], + }, + }, + }, +]; From 3a08c75ca6e7e64a9e48f7421de7fcd146a34bc9 Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:46:44 +0530 Subject: [PATCH 07/26] chore: update transformV2.js --- src/v0/destinations/klaviyo/transformV2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index 5e2cf7f19f..ebf9a94b34 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -187,7 +187,7 @@ const processRouterDestV2 = async (inputs, reqMetadata) => { // if not transformed getEventChunks( { - message: await process(event, reqMetadata), + message: await processV2(event, reqMetadata), metadata: event.metadata, destination, }, From e7d0ea9f0f280d71587b39b3c7a6943dad14dc49 Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:59:29 +0530 Subject: [PATCH 08/26] Update transformV2.js --- src/v0/destinations/klaviyo/transformV2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index ebf9a94b34..dea5fab4a4 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -187,7 +187,7 @@ const processRouterDestV2 = async (inputs, reqMetadata) => { // if not transformed getEventChunks( { - message: await processV2(event, reqMetadata), + message: await processV2(event), metadata: event.metadata, destination, }, From c018707e7de2608d5680c7b57807150365092554 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Wed, 17 Jul 2024 09:35:49 +0530 Subject: [PATCH 09/26] fix: improved quality for batching for v2 --- src/v0/destinations/klaviyo/transformV2.js | 174 +++++++------ src/v0/destinations/klaviyo/util.js | 216 +++++++++++++--- .../destinations/klaviyo/processor/dataV2.ts | 8 +- .../klaviyo/processor/groupTestDataV2.ts | 193 +++++++++++++++ .../klaviyo/processor/identifyTestDataV2.ts | 138 +++++++++++ .../klaviyo/processor/screenTestDataV2.ts | 4 +- .../klaviyo/processor/trackTestDataV2.ts | 232 ++++++++++++++++++ .../klaviyo/processor/validationTestDataV2.ts | 4 +- .../klaviyo/router/commonConfig.ts | 199 +++++++++++++++ .../destinations/klaviyo/router/data.ts | 164 +------------ 10 files changed, 1039 insertions(+), 293 deletions(-) create mode 100644 test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts create mode 100644 test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts create mode 100644 test/integrations/destinations/klaviyo/router/commonConfig.ts diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index dea5fab4a4..a19a8e78cb 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -4,19 +4,22 @@ const get = require('get-value'); const { ConfigurationError, InstrumentationError } = require('@rudderstack/integrations-lib'); const { EventType } = require('../../../constants'); -const { CONFIG_CATEGORIES, BASE_ENDPOINT, MAPPING_CONFIG, revision } = require('./config'); -const { batchSubscribeEvents, constructProfile, subscribeUserToListV2 } = require('./util'); +const { CONFIG_CATEGORIES, MAPPING_CONFIG } = require('./config'); +const { + batchEvents, + constructProfile, + subscribeUserToListV2, + buildRequest, + buildSubscriptionRequest, +} = require('./util'); const { - defaultRequestConfig, constructPayload, getFieldValueFromMessage, - defaultPostRequestConfig, removeUndefinedAndNullValues, getSuccessRespEvents, handleRtTfSingleEventError, flattenJson, } = require('../../util'); -const { JSON_MIME_TYPE } = require('../../util/constant'); /** * Main Identify request handler func @@ -27,33 +30,20 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); * @param {*} message * @param {*} category * @param {*} destination - * @returns + * @returns one object with keys profile and subscription(conditional) and values as objects */ const identifyRequestHandler = (message, category, destination) => { // If listId property is present try to subscribe/member user in list - const { privateApiKey, listId } = destination.Config; + const { listId } = destination.Config; const payload = removeUndefinedAndNullValues(constructProfile(message, destination, true)); - const endpoint = `${BASE_ENDPOINT}${category.apiUrl}`; - const requestOptions = { - headers: { - Authorization: `Klaviyo-API-Key ${privateApiKey}`, - Accept: JSON_MIME_TYPE, - 'Content-Type': JSON_MIME_TYPE, - revision, - }, - }; - const profileRequest = defaultRequestConfig(); - profileRequest.endpoint = endpoint; - profileRequest.body.JSON = payload; - profileRequest.headers = requestOptions.headers; - let responseList = profileRequest; + const response = { profile: payload }; const traitsInfo = getFieldValueFromMessage(message, 'traits'); // check if user wants to subscribe profile or not and listId is present or not if (traitsInfo?.properties?.subscribe && (traitsInfo.properties?.listId || listId)) { - responseList = [responseList, subscribeUserToListV2(message, traitsInfo, destination)]; + response.subscription = subscribeUserToListV2(message, traitsInfo, destination); } // returning list if subscription to a list is to be done else returning an object to upsert profile - return responseList; + return response; }; /** @@ -63,11 +53,10 @@ const identifyRequestHandler = (message, category, destination) => { * @param {*} message * @param {*} category * @param {*} destination - * @returns requestBody + * @returns event request */ const trackOrScreenRequestHandler = (message, category, destination) => { - const payload = {}; - const { privateApiKey, flattenProperties } = destination.Config; + const { flattenProperties } = destination.Config; // event for track and name for screen call const event = get(message, 'event') || get(message, 'name'); if (event && typeof event !== 'string') { @@ -90,18 +79,7 @@ const trackOrScreenRequestHandler = (message, category, destination) => { }, }, }; - payload.data = { type: 'event', attributes }; - const response = defaultRequestConfig(); - response.endpoint = `${BASE_ENDPOINT}${category.apiUrl}`; - response.method = defaultPostRequestConfig.requestMethod; - response.headers = { - Authorization: `Klaviyo-API-Key ${privateApiKey}`, - Accept: JSON_MIME_TYPE, - 'Content-Type': JSON_MIME_TYPE, - revision, - }; - response.body.JSON = removeUndefinedAndNullValues(payload); - return response; + return { event: { data: { type: 'event', attributes } } }; }; /** @@ -110,7 +88,7 @@ const trackOrScreenRequestHandler = (message, category, destination) => { * @param {*} message * @param {*} category * @param {*} destination - * @returns + * @returns subscription object */ const groupRequestHandler = (message, category, destination) => { if (!message.groupId) { @@ -121,10 +99,10 @@ const groupRequestHandler = (message, category, destination) => { throw new InstrumentationError('Subscribe flag should be true for group call'); } - return [subscribeUserToListV2(message, traitsInfo, destination)]; + return { subscription: subscribeUserToListV2(message, traitsInfo, destination) }; }; -const processV2 = async (event) => { +const processEvent = (event) => { const { message, destination } = event; if (!message.type) { throw new InstrumentationError('Event type is required'); @@ -156,63 +134,83 @@ const processV2 = async (event) => { return response; }; -// This function separates subscribe response and other responses in chunks -const getEventChunks = (event, subscribeRespList, nonSubscribeRespList) => { - if (Array.isArray(event.message)) { - // this list contains responses for subscribe endpoint - subscribeRespList.push(event); - } else { - // this list doesn't contain subsribe endpoint responses - nonSubscribeRespList.push(event); +const processV2 = (event) => { + const response = processEvent(event); + const { destination } = event; + const respList = []; + if (response.profile) { + respList.push(buildRequest(response.profile, destination, CONFIG_CATEGORIES.IDENTIFYV2)); + } + if (response.subscription) { + respList.push( + buildSubscriptionRequest(response.subscription, destination, CONFIG_CATEGORIES.TRACKV2), + ); + } + if (response.event) { + respList.push(buildRequest(response.event, destination, CONFIG_CATEGORIES.TRACKV2)); } + return respList; }; -const processRouterDestV2 = async (inputs, reqMetadata) => { +// This function separates subscribe, proifle and event responses from process () and other responses in chunks +const getEventChunks = (input, subscribeRespList, profileRespList, eventRespList) => { + if (input.payload.subscription) { + subscribeRespList.push({ payload: input.payload.subscription, metadata: input.metadata }); + } + if (input.payload.profile) { + profileRespList.push({ payload: input.payload.profile, metadata: input.metadata }); + } + if (input.payload.event) { + eventRespList.push({ payload: input.payload.event, metadata: input.metadata }); + } +}; + +const processRouterDestV2 = (inputs, reqMetadata) => { let batchResponseList = []; const batchErrorRespList = []; const subscribeRespList = []; - const nonSubscribeRespList = []; + const profileRespList = []; + const eventRespList = []; const { destination } = inputs[0]; - await Promise.all( - inputs.map(async (event) => { - try { - if (event.message.statusCode) { - // already transformed event - getEventChunks( - { message: event.message, metadata: event.metadata, destination }, - subscribeRespList, - nonSubscribeRespList, - ); - } else { - // if not transformed - getEventChunks( - { - message: await processV2(event), - metadata: event.metadata, - destination, - }, - subscribeRespList, - nonSubscribeRespList, - ); - } - } catch (error) { - const errRespEvent = handleRtTfSingleEventError(event, error, reqMetadata); - batchErrorRespList.push(errRespEvent); + inputs.map((event) => { + try { + if (event.message.statusCode) { + // already transformed event + getEventChunks( + { message: event.message, metadata: event.metadata, destination }, + subscribeRespList, + profileRespList, + eventRespList, + ); + } else { + // if not transformed + getEventChunks( + { + payload: processEvent(event), + metadata: event.metadata, + }, + subscribeRespList, + profileRespList, + eventRespList, + ); } - }), - ); - const batchedSubscribeResponseList = []; - if (subscribeRespList.length > 0) { - const batchedResponseList = batchSubscribeEvents(subscribeRespList, 'v2'); - batchedSubscribeResponseList.push(...batchedResponseList); - } - const nonSubscribeSuccessList = nonSubscribeRespList.map((resp) => { - const response = resp; - const { message, metadata, destination: eventDestination } = response; - return getSuccessRespEvents(message, [metadata], eventDestination); + } catch (error) { + const errRespEvent = handleRtTfSingleEventError(event, error, reqMetadata); + batchErrorRespList.push(errRespEvent); + } + }); + const batchedResponseList = batchEvents(subscribeRespList, profileRespList, destination); + // building and pushing all the event requests + const eventRequestList = eventRespList.map((resp) => { + const { payload, metadata } = resp; + return getSuccessRespEvents( + buildRequest(payload, destination, CONFIG_CATEGORIES.TRACKV2), + [metadata], + destination, + ); }); - batchResponseList = [...batchedSubscribeResponseList, ...nonSubscribeSuccessList]; + batchResponseList = [...batchedResponseList, ...eventRequestList]; return [...batchResponseList, ...batchErrorRespList]; }; diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index bd6c3b2456..f594fac8fa 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -214,7 +214,7 @@ const populateCustomFieldsFromTraits = (message) => { return customProperties; }; -const generateBatchedPaylaodForArray = (events, version) => { +const generateBatchedPaylaodForArray = (events) => { let batchEventResponse = defaultBatchRequestConfig(); const batchResponseList = []; const metadata = []; @@ -224,10 +224,6 @@ const generateBatchedPaylaodForArray = (events, version) => { events.forEach((ev, index) => { if (index === 0) { batchResponseList.push(ev.message.body.JSON); - } else if (version === 'v2') { - batchResponseList[0].data.attributes.profiles.data.push( - ...ev.message.body.JSON.data.attributes.profiles.data, - ); } else { batchResponseList[0].data.attributes.subscriptions.push( ...ev.message.body.JSON.data.attributes.subscriptions, @@ -266,22 +262,28 @@ const generateBatchedPaylaodForArray = (events, version) => { * @param {*} version this parameter to know the API and accordingly fetch listId from payload from right place * @returns */ -const groupSubscribeResponsesUsingListId = (subscribeResponseList, version) => { - const subscribeEventGroups = lodash.groupBy(subscribeResponseList, (event) => - version === 'v2' - ? event.message.body.JSON.data.relationships.list.data.id - : event.message.body.JSON.data.attributes.list_id, +const groupSubscribeResponsesUsingListId = (subscribeResponseList) => { + const subscribeEventGroups = lodash.groupBy( + subscribeResponseList, + (event) => event.message.body.JSON.data.attributes.list_id, + ); + return subscribeEventGroups; +}; +const groupSubscribeResponsesUsingListIdV2 = (subscribeResponseList) => { + const subscribeEventGroups = lodash.groupBy( + subscribeResponseList, + (event) => event.payload.listId, ); return subscribeEventGroups; }; -const getBatchedResponseList = (subscribeEventGroups, identifyResponseList, version) => { +const getBatchedResponseList = (subscribeEventGroups, identifyResponseList) => { let batchedResponseList = []; Object.keys(subscribeEventGroups).forEach((listId) => { // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] const eventChunks = lodash.chunk(subscribeEventGroups[listId], MAX_BATCH_SIZE); const batchedResponse = eventChunks.map((chunk) => { - const batchEventResponse = generateBatchedPaylaodForArray(chunk, version); + const batchEventResponse = generateBatchedPaylaodForArray(chunk); return getSuccessRespEvents( batchEventResponse.batchedRequest, batchEventResponse.metadata, @@ -291,17 +293,15 @@ const getBatchedResponseList = (subscribeEventGroups, identifyResponseList, vers }); batchedResponseList = [...batchedResponseList, ...batchedResponse]; }); - if (identifyResponseList.length > 0) { identifyResponseList.forEach((response) => { batchedResponseList[0].batchedRequest.push(response); }); } - return batchedResponseList; }; -const batchSubscribeEvents = (subscribeRespList, version = 'v1') => { +const batchSubscribeEvents = (subscribeRespList) => { const identifyResponseList = []; subscribeRespList.forEach((event) => { const processedEvent = event; @@ -316,16 +316,26 @@ const batchSubscribeEvents = (subscribeRespList, version = 'v1') => { } }); - const subscribeEventGroups = groupSubscribeResponsesUsingListId(subscribeRespList, version); + const subscribeEventGroups = groupSubscribeResponsesUsingListId(subscribeRespList); - const batchedResponseList = getBatchedResponseList( - subscribeEventGroups, - identifyResponseList, - version, - ); + const batchedResponseList = getBatchedResponseList(subscribeEventGroups, identifyResponseList); return batchedResponseList; }; +const buildRequest = (payload, destination, category) => { + const { privateApiKey } = destination.Config; + const response = defaultRequestConfig(); + response.endpoint = `${BASE_ENDPOINT}${category.apiUrl}`; + response.method = defaultPostRequestConfig.requestMethod; + response.headers = { + Authorization: `Klaviyo-API-Key ${privateApiKey}`, + Accept: JSON_MIME_TYPE, + 'Content-Type': JSON_MIME_TYPE, + revision, + }; + response.body.JSON = removeUndefinedAndNullValues(payload); + return response; +}; /** * This function generates the metadat object used for updating a list attribute and unset properties @@ -456,12 +466,12 @@ const constructProfile = (message, destination, isIdentifyCall) => { }; /** - * This function is used for creating response for subscribing users to a particular list for V2 + * This function is used for creating profile response for subscribing users to a particular list for V2 * DOCS: https://developers.klaviyo.com/en/reference/subscribe_profiles */ const subscribeUserToListV2 = (message, traitsInfo, destination) => { // listId from message properties are preferred over Config listId - const { privateApiKey, consent } = destination.Config; + const { consent } = destination.Config; let { listId } = destination.Config; let subscribeConsent = traitsInfo?.properties?.consent || consent; const email = getFieldValueFromMessage(message, 'email'); @@ -501,31 +511,160 @@ const subscribeUserToListV2 = (message, traitsInfo, destination) => { listId = message.groupId; } - const payload = { - data: { - type: 'profile-subscription-bulk-create-job', - attributes: { profiles: { data: [profile] } }, - relationships: { - list: { - data: { - type: 'list', - id: listId, - }, + return { listId, profile: [profile] }; +}; +/** + * This Create a subscription payload to subscribe profile(s) to list listId + * @param {*} listId + * @param {*} profile + */ +const getSubscriptionPayload = (listId, profile) => ({ + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { profiles: { data: profile } }, + relationships: { + list: { + data: { + type: 'list', + id: listId, }, }, }, + }, +}); + +/** + * This function takes susbscriptions as input and batches them into a single request body + * @param {events} + * events= [ + * { payload: {id:'list_id', profile: {}}, metadata:{} }, + * { payload: {id:'list_id', profile: {}}, metadata:{} } + * ] + */ + +const generateBatchedSubscriptionRequest = (events, destination) => { + const batchEventResponse = defaultBatchRequestConfig(); + const metadata = []; + // fetching listId from first event as listId is same for all the events + const listId = events[0].payload?.id; + const profiles = []; // list of profiles to be subscribes + // Batch profiles into dest batch structure + events.forEach((ev) => { + profiles.push(ev.payload.profile); + metadata.push(ev.metadata); + }); + + batchEventResponse.batchedRequest = Object.values(batchEventResponse); + batchEventResponse.batchedRequest[0].body.JSON = getSubscriptionPayload(listId, profiles); + + batchEventResponse.batchedRequest[0].endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; + + batchEventResponse.batchedRequest[0].headers = { + Authorization: `Klaviyo-API-Key ${destination.Config.privateApiKey}`, + 'Content-Type': JSON_MIME_TYPE, + Accept: JSON_MIME_TYPE, + revision, }; + + return { + ...batchEventResponse, + metadata, + destination, + }; +}; + +/** + * This function fetches the profileRequests with metadata present in metadata array build a request for them + * and add these requests batchEvent Response + * @param {*} profileReq array of profile requests + * @param {*} metadataArray array of metadata + * @param {*} batchEventResponse + */ +const updateBatchEventResponseWithProfileRequests = ( + profileReq, + subscriptionMetadataArray, + batchEventResponse, +) => { + const subscriptionListJobIds = subscriptionMetadataArray.map((metadata) => metadata.jobId); + profileReq.forEach((profile) => { + if (profile.metadata.jobId in subscriptionListJobIds) { + batchEventResponse.batchedRequest.push( + buildRequest(profile.payload, batchEventResponse.destination, CONFIG_CATEGORIES.IDENTIFYV2), + ); + } + }); +}; + +/** + * This function returns the list of profileReq which do not metadata common with subcriptionMetadataArray + * @param {*} profileReq + * @param {*} subscriptionMetadataArray + * @returns + */ +const getRemainingProfiles = (profileReq, subscriptionMetadataArray) => { + const subscriptionListJobIds = subscriptionMetadataArray.map((metadata) => metadata.jobId); + return profileReq.filter((profile) => !(profile.metadata.jobId in subscriptionListJobIds)); +}; +/** + * This function batches the requests. Alogorithm + * Batch events from Subscribe Resp List having same listId/groupId to be subscribed and have their metadata array + * For this metadata array get all profileRequests and add them prior to batched Subscribe Request in the same batched Request + * Make another batched request for the remaning profile requests and another for all the event requests + * @param {*} subscribeRespList + * @param {*} profileRespList + * @param {*} eventRespList + */ +const batchEvents = (subscribeRespList, profileRespList, destination) => { + const batchedResponseList = []; + let remainingProfileReq = profileRespList; + const subscribeEventGroups = groupSubscribeResponsesUsingListIdV2(subscribeRespList); + Object.keys(subscribeEventGroups).forEach((listId) => { + // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] + const eventChunks = lodash.chunk(subscribeEventGroups[listId], MAX_BATCH_SIZE); + const batchedResponse = eventChunks.map((chunk) => { + const batchEventResponse = generateBatchedSubscriptionRequest(chunk, destination); + const { metadata: subscriptionMetadataArray, batchedRequest } = batchEventResponse; + updateBatchEventResponseWithProfileRequests( + remainingProfileReq, + subscriptionMetadataArray, + batchEventResponse, + ); + remainingProfileReq = getRemainingProfiles(remainingProfileReq, subscriptionMetadataArray); + return getSuccessRespEvents(batchedRequest, subscriptionMetadataArray, destination, true); + }); + batchedResponseList.push(...batchedResponse); + }); + const profiles = []; + // push profiles for which there is no subscription + remainingProfileReq.forEach((input) => { + profiles.push( + getSuccessRespEvents( + buildRequest(input.payload, destination, CONFIG_CATEGORIES.IDENTIFYV2), + [input.metadata], + destination, + ), + ); + }); + return [...profiles, ...batchedResponseList]; +}; + +/** + * This function accepts subscriptions object and builds a request for it + * @param {*} subscription + * @param {*} destination + * @returns defaultRequestConfig + */ +const buildSubscriptionRequest = (subscription, destination) => { const response = defaultRequestConfig(); - response.method = defaultPostRequestConfig.requestMethod; response.endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; + response.method = defaultPostRequestConfig.requestMethod; response.headers = { - Authorization: `Klaviyo-API-Key ${privateApiKey}`, + Authorization: `Klaviyo-API-Key ${destination.Config.privateApiKey}`, Accept: JSON_MIME_TYPE, 'Content-Type': JSON_MIME_TYPE, revision, }; - response.body.JSON = removeUndefinedAndNullValues(payload); - + response.body.JSON = getSubscriptionPayload(subscription.listId, subscription.profile); return response; }; @@ -540,4 +679,7 @@ module.exports = { constructProfile, subscribeUserToListV2, getProfileMetadataAndMetadataFields, + batchEvents, + buildRequest, + buildSubscriptionRequest, }; diff --git a/test/integrations/destinations/klaviyo/processor/dataV2.ts b/test/integrations/destinations/klaviyo/processor/dataV2.ts index 118a41164e..32c3c4206a 100644 --- a/test/integrations/destinations/klaviyo/processor/dataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/dataV2.ts @@ -1,13 +1,13 @@ -// import { groupTestData } from './groupTestDataV2'; +import { groupTestData } from './groupTestDataV2'; import { identifyData } from './identifyTestDataV2'; import { screenTestData } from './screenTestDataV2'; -// import { trackTestData } from './trackTestDataV2'; +import { trackTestData } from './trackTestDataV2'; import { validationTestData } from './validationTestData'; export const dataV2 = [ ...identifyData, - // ...trackTestData, + ...trackTestData, ...screenTestData, - // ...groupTestData, + ...groupTestData, ...validationTestData, ]; diff --git a/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts new file mode 100644 index 0000000000..8a944b76df --- /dev/null +++ b/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts @@ -0,0 +1,193 @@ +import { Destination } from '../../../../../src/types'; +import { ProcessorTestData } from '../../../testTypes'; +import { + generateMetadata, + generateSimplifiedGroupPayload, + transformResultBuilder, +} from '../../../testUtils'; + +const destination: Destination = { + ID: '123', + Name: 'klaviyo', + DestinationDefinition: { + ID: '123', + Name: 'klaviyo', + DisplayName: 'klaviyo', + Config: {}, + }, + Config: { + version: 'v2', + privateApiKey: 'dummyPrivateApiKey', + consent: ['email'], + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; + +const headers = { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + 'Content-Type': 'application/json', + Accept: 'application/json', + revision: '2024-06-15', +}; + +const endpoint = 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs'; + +const commonOutputSubscriptionProps = { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'test@rudderstack.com', + phone_number: '+12 345 678 900', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + ], + }, +}; + +const subscriptionRelations = { + list: { + data: { + type: 'list', + id: 'group_list_id', + }, + }, +}; + +export const groupTestData: ProcessorTestData[] = [ + { + id: 'klaviyo-group-test-1', + name: 'klaviyo', + description: 'Simple group call', + scenario: 'Business', + successCriteria: + 'Response should contain only group payload and status code should be 200, for the group payload a subscription payload should be present in the final payload with email and phone', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: generateSimplifiedGroupPayload({ + userId: 'user123', + groupId: 'group_list_id', + traits: { + subscribe: true, + }, + context: { + traits: { + email: 'test@rudderstack.com', + phone: '+12 345 678 900', + consent: ['email'], + }, + }, + timestamp: '2020-01-21T00:21:34.208Z', + }), + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: commonOutputSubscriptionProps, + relationships: subscriptionRelations, + }, + }, + a: { + data: { + attributes: { + list_id: 'XUepkK', + subscriptions: [ + { email: 'test@rudderstack.com', phone_number: '+12 345 678 900' }, + ], + }, + type: 'profile-subscription-bulk-create-job', + }, + }, + endpoint: endpoint, + headers: headers, + method: 'POST', + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'klaviyo-group-test-2', + name: 'klaviyo', + description: 'Simple group call without groupId', + scenario: 'Business', + successCriteria: + 'Response should contain error message and status code should be 400, as we are not sending groupId in the payload and groupId is a required field for group events', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: generateSimplifiedGroupPayload({ + userId: 'user123', + groupId: '', + traits: { + subscribe: true, + }, + context: { + traits: { + email: 'test@rudderstack.com', + phone: '+12 345 678 900', + consent: 'email', + }, + }, + timestamp: '2020-01-21T00:21:34.208Z', + }), + metadata: generateMetadata(2), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: 'groupId is a required field for group events', + statTags: { + destType: 'KLAVIYO', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + }, + statusCode: 400, + metadata: generateMetadata(2), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts index 65356e9031..830dffec47 100644 --- a/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts @@ -291,4 +291,142 @@ export const identifyData: ProcessorTestData[] = [ }, }, }, + { + id: 'klaviyo-identify-150624-test-3', + name: 'klaviyo', + description: + '150624 -> Identify call without email and phone & enforceEmailAsPrimary enabled from UI', + scenario: 'Business', + successCriteria: + 'Response should contain error message and status code should be 400, as we are not sending email and phone in the payload and enforceEmailAsPrimary is enabled from UI', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { enforceEmailAsPrimary: true }), + message: generateSimplifiedIdentifyPayload({ + sentAt, + userId, + context: { + traits: removeUndefinedAndNullValues({ + ...commonTraits, + email: undefined, + phone: undefined, + }), + }, + anonymousId, + originalTimestamp, + }), + metadata: generateMetadata(7), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: 'None of email and phone are present in the payload', + statTags: { + destType: 'KLAVIYO', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + }, + statusCode: 400, + metadata: generateMetadata(7), + }, + ], + }, + }, + }, + { + id: 'klaviyo-identify-150624-test-5', + name: 'klaviyo', + description: '150624 -> Identify call with enforceEmailAsPrimary enabled in destination config', + scenario: 'Business', + successCriteria: + 'Response should contain two payloads one for profile and other for subscription, response status code should be 200, for the profile updation payload there should be no external_id field in the payload as enforceEmailAsPrimary is set to true in the destination config and the userId should be mapped to _id field in the properties object', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { enforceEmailAsPrimary: true }), + message: generateSimplifiedIdentifyPayload({ + sentAt, + userId, + context: { + traits: commonTraits, + }, + anonymousId, + originalTimestamp, + }), + metadata: generateMetadata(5), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers: commonOutputHeaders, + JSON: { + data: { + type: 'profile', + attributes: removeUndefinedAndNullValues({ + ...commonOutputUserProps, + properties: { + ...commonOutputUserProps.properties, + _id: userId, + }, + // remove external_id from the payload + external_id: undefined, + }), + meta: { + patch_properties: {}, + }, + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(5), + }, + { + output: transformResultBuilder({ + userId: '', + method: 'POST', + endpoint: subscribeEndpoint, + headers: commonOutputHeaders, + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: commonOutputSubscriptionProps, + relationships: subscriptionRelations, + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(5), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts index 41caa1d1bb..3ccafb39a6 100644 --- a/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts @@ -26,9 +26,9 @@ const destination: Destination = { export const screenTestData: ProcessorTestData[] = [ { - id: 'klaviyo-screen-test-1', + id: 'klaviyo-screen-150624-test-1', name: 'klaviyo', - description: 'Screen event call with properties and contextual traits', + description: '150624 -> Screen event call with properties and contextual traits', scenario: 'Business', successCriteria: 'Response should contain only event payload and status code should be 200, for the event payload should contain properties and contextual traits in the payload', diff --git a/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts new file mode 100644 index 0000000000..f9e5a0233e --- /dev/null +++ b/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts @@ -0,0 +1,232 @@ +import { Destination } from '../../../../../src/types'; +import { ProcessorTestData } from '../../../testTypes'; +import { + generateMetadata, + generateSimplifiedTrackPayload, + generateTrackPayload, + overrideDestination, + transformResultBuilder, +} from '../../../testUtils'; + +const destination: Destination = { + ID: '123', + Name: 'klaviyo', + DestinationDefinition: { + ID: '123', + Name: 'klaviyo', + DisplayName: 'klaviyo', + Config: {}, + }, + Config: { + privateApiKey: 'dummyPrivateApiKey', + version: 'v2', + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; + +const commonTraits = { + id: 'user@1', + age: '22', + anonymousId: '9c6bd77ea9da3e68', +}; + +const commonProps = { + PreviouslVicePresident: true, + YearElected: 1801, + VicePresidents: ['AaronBurr', 'GeorgeClinton'], +}; + +const commonOutputHeaders = { + Accept: 'application/json', + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + 'Content-Type': 'application/json', + revision: '2024-06-15', +}; +const profileAttributes = { + email: 'test@rudderstack.com', + phone_number: '9112340375', + external_id: 'sajal12', + anonymous_id: '9c6bd77ea9da3e68', + properties: { + age: '22', + name: 'Test', + description: 'Sample description', + id: 'user@1', + }, + meta: { + patch_properties: {}, + }, +}; +const eventEndPoint = 'https://a.klaviyo.com/api/events'; + +export const trackTestData: ProcessorTestData[] = [ + { + id: 'klaviyo-track-150624-test-1', + name: 'klaviyo', + description: '150624 -> Simple track event call', + scenario: 'Business', + successCriteria: + 'Response should contain profile and event payload and status code should be 200, for the event payload should contain contextual traits and properties in the payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'TestEven002', + sentAt: '2021-01-25T16:12:02.048Z', + userId: 'sajal12', + context: { + traits: { + ...commonTraits, + name: 'Test', + email: 'test@rudderstack.com', + phone: '9112340375', + description: 'Sample description', + }, + }, + properties: { + ...commonProps, + revenue: 3000, + currency: 'USD', + }, + anonymousId: '9c6bd77ea9da3e68', + originalTimestamp: '2021-01-25T15:32:56.409Z', + }), + metadata: generateMetadata(2), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: eventEndPoint, + headers: commonOutputHeaders, + JSON: { + data: { + type: 'event', + attributes: { + metric: { + data: { + type: 'metric', + attributes: { + name: 'TestEven002', + }, + }, + }, + profile: { + data: { + type: 'profile', + attributes: profileAttributes, + }, + }, + properties: commonProps, + value: 3000, + value_currency: 'USD', + time: '2021-01-25T15:32:56.409Z', + }, + }, + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(2), + }, + ], + }, + }, + }, + { + id: 'klaviyo-track-150624-test-2', + name: 'klaviyo', + description: + "150624 -> Track event call, with make email or phone as primary identifier toggle on but it doesn't matter as this toggle is only for identify call", + scenario: 'Business', + successCriteria: + 'Response should contain only event payload with profile object and status code should be 200, for the event payload should contain contextual traits and properties in the payload, and email should be mapped to email and userId should be mapped to external_id as usual', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { enforceEmailAsPrimary: true }), + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'TestEven001', + sentAt: '2021-01-25T16:12:02.048Z', + userId: 'sajal12', + context: { + traits: { + ...commonTraits, + description: 'Sample description', + name: 'Test', + email: 'test@rudderstack.com', + phone: '9112340375', + }, + }, + properties: commonProps, + anonymousId: '9c6bd77ea9da3e68', + originalTimestamp: '2021-01-25T15:32:56.409Z', + }), + metadata: generateMetadata(3), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: eventEndPoint, + headers: commonOutputHeaders, + JSON: { + data: { + type: 'event', + attributes: { + metric: { + data: { + type: 'metric', + attributes: { + name: 'TestEven001', + }, + }, + }, + properties: commonProps, + profile: { + data: { + type: 'profile', + attributes: { + ...profileAttributes, + }, + }, + }, + time: '2021-01-25T15:32:56.409Z', + }, + }, + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(3), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts index 9d673e8fc7..613356c9b5 100644 --- a/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts @@ -22,9 +22,9 @@ const destination: Destination = { export const validationTestData: ProcessorTestData[] = [ { - id: 'klaviyo-validation-test-1', + id: 'klaviyo-validation-150624-test-1', name: 'klaviyo', - description: '[Error]: Check for unsupported message type', + description: '150624->[Error]: Check for unsupported message type', scenario: 'Framework', successCriteria: 'Response should contain error message and status code should be 400, as we are sending a message type which is not supported by Klaviyo destination and the error message should be Event type random is not supported', diff --git a/test/integrations/destinations/klaviyo/router/commonConfig.ts b/test/integrations/destinations/klaviyo/router/commonConfig.ts new file mode 100644 index 0000000000..0abf4ab055 --- /dev/null +++ b/test/integrations/destinations/klaviyo/router/commonConfig.ts @@ -0,0 +1,199 @@ +import { generateMetadata } from '../../../testUtils'; +import { Destination, RouterTransformationRequest } from '../../../../../src/types'; +const destination: Destination = { + ID: '123', + Name: 'klaviyo', + DestinationDefinition: { + ID: '123', + Name: 'klaviyo', + DisplayName: 'klaviyo', + Config: {}, + }, + Config: { + privateApiKey: 'dummyPrivateApiKey', + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; +const destinationV2: Destination = { + ID: '123', + Name: 'klaviyo', + DestinationDefinition: { + ID: '123', + Name: 'klaviyo', + DisplayName: 'klaviyo', + Config: {}, + }, + Config: { + privateApiKey: 'dummyPrivateApiKey', + version: 'v2', + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; +const getRequest = (version) => { + return [ + { + destination: version === 'v2' ? destinationV2 : destination, + metadata: generateMetadata(1), + message: { + type: 'identify', + sentAt: '2021-01-03T17:02:53.195Z', + userId: 'test', + channel: 'web', + context: { + os: { name: '', version: '' }, + app: { + name: 'RudderLabs JavaScript SDK', + build: '1.0.0', + version: '1.1.11', + namespace: 'com.rudderlabs.javascript', + }, + traits: { + firstName: 'Test', + lastName: 'Rudderlabs', + email: 'test_1@rudderstack.com', + phone: '+12 345 578 900', + userId: 'Testc', + title: 'Developer', + organization: 'Rudder', + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + Flagged: false, + Residence: 'Shibuya', + properties: { consent: ['email', 'sms'] }, + }, + locale: 'en-US', + screen: { density: 2 }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.1.11' }, + campaign: {}, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', + }, + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + integrations: { All: true }, + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + { + destination: version === 'v2' ? destinationV2 : destination, + metadata: generateMetadata(2), + message: { + type: 'identify', + sentAt: '2021-01-03T17:02:53.195Z', + userId: 'test', + channel: 'web', + context: { + os: { name: '', version: '' }, + app: { + name: 'RudderLabs JavaScript SDK', + build: '1.0.0', + version: '1.1.11', + namespace: 'com.rudderlabs.javascript', + }, + traits: { + firstName: 'Test', + lastName: 'Rudderlabs', + email: 'test@rudderstack.com', + phone: '+12 345 578 900', + userId: 'test', + title: 'Developer', + organization: 'Rudder', + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + Flagged: false, + Residence: 'Shibuya', + properties: { listId: 'XUepkK', subscribe: true, consent: ['email', 'sms'] }, + }, + locale: 'en-US', + screen: { density: 2 }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.1.11' }, + campaign: {}, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', + }, + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + integrations: { All: true }, + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + { + destination: version === 'v2' ? destinationV2 : destination, + metadata: generateMetadata(3), + message: { + userId: 'user123', + type: 'group', + groupId: 'XUepkK', + traits: { subscribe: true }, + context: { + traits: { + email: 'test@rudderstack.com', + phone: '+12 345 678 900', + consent: ['email'], + }, + ip: '14.5.67.21', + library: { name: 'http' }, + }, + timestamp: '2020-01-21T00:21:34.208Z', + }, + }, + { + destination: version === 'v2' ? destinationV2 : destination, + metadata: generateMetadata(4), + message: { + userId: 'user123', + type: 'random', + groupId: 'XUepkK', + traits: { subscribe: true }, + context: { + traits: { + email: 'test@rudderstack.com', + phone: '+12 345 678 900', + consent: 'email', + }, + ip: '14.5.67.21', + library: { name: 'http' }, + }, + timestamp: '2020-01-21T00:21:34.208Z', + }, + }, + { + destination: version === 'v2' ? destinationV2 : destination, + metadata: generateMetadata(5), + message: { + userId: 'user123', + type: 'group', + groupId: '', + traits: { subscribe: true }, + context: { + traits: { + email: 'test@rudderstack.com', + phone: '+12 345 678 900', + consent: 'email', + }, + ip: '14.5.67.21', + library: { name: 'http' }, + }, + timestamp: '2020-01-21T00:21:34.208Z', + }, + }, + ]; +}; +export const routerRequest: RouterTransformationRequest = { + input: getRequest('v1'), + destType: 'klaviyo', +}; +export const routerRequestV2: RouterTransformationRequest = { + input: getRequest('v2'), + destType: 'klaviyo', +}; diff --git a/test/integrations/destinations/klaviyo/router/data.ts b/test/integrations/destinations/klaviyo/router/data.ts index 8866a8a546..73bbd24a3b 100644 --- a/test/integrations/destinations/klaviyo/router/data.ts +++ b/test/integrations/destinations/klaviyo/router/data.ts @@ -1,6 +1,8 @@ import { Destination, RouterTransformationRequest } from '../../../../../src/types'; import { RouterTestData } from '../../../testTypes'; +import { routerRequest } from './commonConfig'; import { generateMetadata } from '../../../testUtils'; +import { dataV2 } from './dataV2'; const destination: Destination = { ID: '123', @@ -12,7 +14,6 @@ const destination: Destination = { Config: {}, }, Config: { - publicApiKey: 'dummyPublicApiKey', privateApiKey: 'dummyPrivateApiKey', }, Enabled: true, @@ -20,164 +21,6 @@ const destination: Destination = { Transformations: [], }; -const routerRequest: RouterTransformationRequest = { - input: [ - { - destination, - metadata: generateMetadata(1), - message: { - type: 'identify', - sentAt: '2021-01-03T17:02:53.195Z', - userId: 'test', - channel: 'web', - context: { - os: { name: '', version: '' }, - app: { - name: 'RudderLabs JavaScript SDK', - build: '1.0.0', - version: '1.1.11', - namespace: 'com.rudderlabs.javascript', - }, - traits: { - firstName: 'Test', - lastName: 'Rudderlabs', - email: 'test@rudderstack.com', - phone: '+12 345 578 900', - userId: 'Testc', - title: 'Developer', - organization: 'Rudder', - city: 'Tokyo', - region: 'Kanto', - country: 'JP', - zip: '100-0001', - Flagged: false, - Residence: 'Shibuya', - properties: { consent: ['email', 'sms'] }, - }, - locale: 'en-US', - screen: { density: 2 }, - library: { name: 'RudderLabs JavaScript SDK', version: '1.1.11' }, - campaign: {}, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', - }, - rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', - messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', - anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', - integrations: { All: true }, - originalTimestamp: '2021-01-03T17:02:53.193Z', - }, - }, - { - destination, - metadata: generateMetadata(2), - message: { - type: 'identify', - sentAt: '2021-01-03T17:02:53.195Z', - userId: 'test', - channel: 'web', - context: { - os: { name: '', version: '' }, - app: { - name: 'RudderLabs JavaScript SDK', - build: '1.0.0', - version: '1.1.11', - namespace: 'com.rudderlabs.javascript', - }, - traits: { - firstName: 'Test', - lastName: 'Rudderlabs', - email: 'test@rudderstack.com', - phone: '+12 345 578 900', - userId: 'test', - title: 'Developer', - organization: 'Rudder', - city: 'Tokyo', - region: 'Kanto', - country: 'JP', - zip: '100-0001', - Flagged: false, - Residence: 'Shibuya', - properties: { listId: 'XUepkK', subscribe: true, consent: ['email', 'sms'] }, - }, - locale: 'en-US', - screen: { density: 2 }, - library: { name: 'RudderLabs JavaScript SDK', version: '1.1.11' }, - campaign: {}, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', - }, - rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', - messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', - anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', - integrations: { All: true }, - originalTimestamp: '2021-01-03T17:02:53.193Z', - }, - }, - { - destination, - metadata: generateMetadata(3), - message: { - userId: 'user123', - type: 'group', - groupId: 'XUepkK', - traits: { subscribe: true }, - context: { - traits: { - email: 'test@rudderstack.com', - phone: '+12 345 678 900', - consent: ['email'], - }, - ip: '14.5.67.21', - library: { name: 'http' }, - }, - timestamp: '2020-01-21T00:21:34.208Z', - }, - }, - { - destination, - metadata: generateMetadata(4), - message: { - userId: 'user123', - type: 'random', - groupId: 'XUepkK', - traits: { subscribe: true }, - context: { - traits: { - email: 'test@rudderstack.com', - phone: '+12 345 678 900', - consent: 'email', - }, - ip: '14.5.67.21', - library: { name: 'http' }, - }, - timestamp: '2020-01-21T00:21:34.208Z', - }, - }, - { - destination, - metadata: generateMetadata(5), - message: { - userId: 'user123', - type: 'group', - groupId: '', - traits: { subscribe: true }, - context: { - traits: { - email: 'test@rudderstack.com', - phone: '+12 345 678 900', - consent: 'email', - }, - ip: '14.5.67.21', - library: { name: 'http' }, - }, - timestamp: '2020-01-21T00:21:34.208Z', - }, - }, - ], - destType: 'klaviyo', -}; - export const data: RouterTestData[] = [ { id: 'klaviyo-router-test-1', @@ -301,7 +144,7 @@ export const data: RouterTestData[] = [ type: 'profile', attributes: { external_id: 'test', - email: 'test@rudderstack.com', + email: 'test_1@rudderstack.com', first_name: 'Test', last_name: 'Rudderlabs', phone_number: '+12 345 578 900', @@ -368,4 +211,5 @@ export const data: RouterTestData[] = [ }, }, }, + // ...dataV2, ]; From 7bd43a0bcb3fcf3833e7ba260046f3a130a7c236 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Wed, 17 Jul 2024 11:20:01 +0530 Subject: [PATCH 10/26] chore: add router test case --- src/v0/destinations/klaviyo/transformV2.js | 15 +- src/v0/destinations/klaviyo/util.js | 38 ++- .../klaviyo/router/commonConfig.ts | 12 +- .../destinations/klaviyo/router/data.ts | 2 +- .../destinations/klaviyo/router/dataV2.ts | 237 ++++++++++++++++++ 5 files changed, 279 insertions(+), 25 deletions(-) create mode 100644 test/integrations/destinations/klaviyo/router/dataV2.ts diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index a19a8e78cb..08ac11d33c 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -11,12 +11,12 @@ const { subscribeUserToListV2, buildRequest, buildSubscriptionRequest, + getTrackRequests, } = require('./util'); const { constructPayload, getFieldValueFromMessage, removeUndefinedAndNullValues, - getSuccessRespEvents, handleRtTfSingleEventError, flattenJson, } = require('../../util'); @@ -200,17 +200,10 @@ const processRouterDestV2 = (inputs, reqMetadata) => { } }); const batchedResponseList = batchEvents(subscribeRespList, profileRespList, destination); - // building and pushing all the event requests - const eventRequestList = eventRespList.map((resp) => { - const { payload, metadata } = resp; - return getSuccessRespEvents( - buildRequest(payload, destination, CONFIG_CATEGORIES.TRACKV2), - [metadata], - destination, - ); - }); + const { anonymousTracking, identifiedTracking } = getTrackRequests(eventRespList, destination); - batchResponseList = [...batchedResponseList, ...eventRequestList]; + // We are doing to maintain event ordering basically once a user is identified klaviyo does not allow user tracking based upon anonymous_id only + batchResponseList = [...anonymousTracking, ...batchedResponseList, ...identifiedTracking]; return [...batchResponseList, ...batchErrorRespList]; }; diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index f594fac8fa..b24f8aa14b 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -259,7 +259,6 @@ const generateBatchedPaylaodForArray = (events) => { /** * It takes list of subscribe responses and groups them on the basis of listId * @param {*} subscribeResponseList - * @param {*} version this parameter to know the API and accordingly fetch listId from payload from right place * @returns */ const groupSubscribeResponsesUsingListId = (subscribeResponseList) => { @@ -473,7 +472,7 @@ const subscribeUserToListV2 = (message, traitsInfo, destination) => { // listId from message properties are preferred over Config listId const { consent } = destination.Config; let { listId } = destination.Config; - let subscribeConsent = traitsInfo?.properties?.consent || consent; + let subscribeConsent = traitsInfo.consent || traitsInfo.properties?.consent || consent; const email = getFieldValueFromMessage(message, 'email'); const phone = getFieldValueFromMessage(message, 'phone'); const profileAttributes = { @@ -546,11 +545,11 @@ const generateBatchedSubscriptionRequest = (events, destination) => { const batchEventResponse = defaultBatchRequestConfig(); const metadata = []; // fetching listId from first event as listId is same for all the events - const listId = events[0].payload?.id; + const listId = events[0].payload?.listId; const profiles = []; // list of profiles to be subscribes // Batch profiles into dest batch structure events.forEach((ev) => { - profiles.push(ev.payload.profile); + profiles.push(...ev.payload.profile); metadata.push(ev.metadata); }); @@ -586,13 +585,15 @@ const updateBatchEventResponseWithProfileRequests = ( batchEventResponse, ) => { const subscriptionListJobIds = subscriptionMetadataArray.map((metadata) => metadata.jobId); + const profilesRequests = []; profileReq.forEach((profile) => { - if (profile.metadata.jobId in subscriptionListJobIds) { - batchEventResponse.batchedRequest.push( + if (subscriptionListJobIds.includes(profile.metadata.jobId)) { + profilesRequests.push( buildRequest(profile.payload, batchEventResponse.destination, CONFIG_CATEGORIES.IDENTIFYV2), ); } }); + batchEventResponse.batchedRequest.unshift(...profilesRequests); }; /** @@ -603,7 +604,7 @@ const updateBatchEventResponseWithProfileRequests = ( */ const getRemainingProfiles = (profileReq, subscriptionMetadataArray) => { const subscriptionListJobIds = subscriptionMetadataArray.map((metadata) => metadata.jobId); - return profileReq.filter((profile) => !(profile.metadata.jobId in subscriptionListJobIds)); + return profileReq.filter((profile) => !subscriptionListJobIds.includes(profile.metadata.jobId)); }; /** * This function batches the requests. Alogorithm @@ -668,6 +669,28 @@ const buildSubscriptionRequest = (subscription, destination) => { return response; }; +const getTrackRequests = (eventRespList, destination) => { + // building and pushing all the event requests + const anonymousTracking = []; + const identifiedTracking = []; + eventRespList.forEach((resp) => { + const { payload, metadata } = resp; + const { attributes: profileAttributes } = payload.data.attributes.profile.attributes; + // eslint-disable-next-line @typescript-eslint/naming-convention + const { email, phone_number, external_id } = profileAttributes; + const request = getSuccessRespEvents( + buildRequest(payload, destination, CONFIG_CATEGORIES.TRACKV2), + [metadata], + destination, + ); + if (email || phone_number || external_id) { + identifiedTracking.push(request); + } else { + anonymousTracking.push(request); + } + }); + return { anonymousTracking, identifiedTracking }; +}; module.exports = { subscribeUserToList, createCustomerProperties, @@ -682,4 +705,5 @@ module.exports = { batchEvents, buildRequest, buildSubscriptionRequest, + getTrackRequests, }; diff --git a/test/integrations/destinations/klaviyo/router/commonConfig.ts b/test/integrations/destinations/klaviyo/router/commonConfig.ts index 0abf4ab055..8d9e1c2d06 100644 --- a/test/integrations/destinations/klaviyo/router/commonConfig.ts +++ b/test/integrations/destinations/klaviyo/router/commonConfig.ts @@ -134,13 +134,13 @@ const getRequest = (version) => { userId: 'user123', type: 'group', groupId: 'XUepkK', - traits: { subscribe: true }, + traits: { + email: 'test@rudderstack.com', + phone: '+12 345 678 900', + subscribe: true, + consent: ['email'], + }, context: { - traits: { - email: 'test@rudderstack.com', - phone: '+12 345 678 900', - consent: ['email'], - }, ip: '14.5.67.21', library: { name: 'http' }, }, diff --git a/test/integrations/destinations/klaviyo/router/data.ts b/test/integrations/destinations/klaviyo/router/data.ts index 73bbd24a3b..8e8c507f48 100644 --- a/test/integrations/destinations/klaviyo/router/data.ts +++ b/test/integrations/destinations/klaviyo/router/data.ts @@ -211,5 +211,5 @@ export const data: RouterTestData[] = [ }, }, }, - // ...dataV2, + ...dataV2, ]; diff --git a/test/integrations/destinations/klaviyo/router/dataV2.ts b/test/integrations/destinations/klaviyo/router/dataV2.ts new file mode 100644 index 0000000000..ff8f1679b2 --- /dev/null +++ b/test/integrations/destinations/klaviyo/router/dataV2.ts @@ -0,0 +1,237 @@ +import { Destination } from '../../../../../src/types'; +import { RouterTestData } from '../../../testTypes'; +import { generateMetadata } from '../../../testUtils'; +import { routerRequestV2 } from './commonConfig'; +const destination: Destination = { + ID: '123', + Name: 'klaviyo', + DestinationDefinition: { + ID: '123', + Name: 'klaviyo', + DisplayName: 'klaviyo', + Config: {}, + }, + Config: { + privateApiKey: 'dummyPrivateApiKey', + version: 'v2', + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; +const userProfileCommonEndpoint = 'https://a.klaviyo.com/api/profile-import'; + +const headers = { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + 'Content-Type': 'application/json', + Accept: 'application/json', + revision: '2024-06-15', +}; +const subscriptionRelations = { + list: { + data: { + type: 'list', + id: 'XUepkK', + }, + }, +}; + +export const dataV2: RouterTestData[] = [ + { + id: 'klaviyo-router-150624-test-1', + name: 'klaviyo', + description: '150624 -> Basic Router Test to test multiple payloads', + scenario: 'Framework', + successCriteria: 'All the subscription events should be batched', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequestV2, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'test', + email: 'test_1@rudderstack.com', + first_name: 'Test', + last_name: 'Rudderlabs', + phone_number: '+12 345 578 900', + anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + title: 'Developer', + organization: 'Rudder', + location: { + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + }, + properties: { Flagged: false, Residence: 'Shibuya' }, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1)], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'test', + email: 'test@rudderstack.com', + first_name: 'Test', + last_name: 'Rudderlabs', + phone_number: '+12 345 578 900', + anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + title: 'Developer', + organization: 'Rudder', + location: { + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + }, + properties: { Flagged: false, Residence: 'Shibuya' }, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'test@rudderstack.com', + phone_number: '+12 345 578 900', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + sms: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + { + type: 'profile', + attributes: { + email: 'test@rudderstack.com', + phone_number: '+12 345 678 900', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + ], + }, + }, + relationships: subscriptionRelations, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(2), generateMetadata(3)], + batched: true, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(4)], + batched: false, + statusCode: 400, + error: 'Event type random is not supported', + statTags: { + destType: 'KLAVIYO', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'native', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + }, + destination, + }, + { + metadata: [generateMetadata(5)], + batched: false, + statusCode: 400, + error: 'groupId is a required field for group events', + statTags: { + destType: 'KLAVIYO', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'native', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + }, + destination, + }, + ], + }, + }, + }, + }, +]; From 1d99f2b8bdf2be61292ef82d713460c07e288692 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Wed, 17 Jul 2024 14:04:09 +0530 Subject: [PATCH 11/26] chore: small fix for track call and add util test --- src/v0/destinations/klaviyo/transformV2.js | 10 ++-- src/v0/destinations/klaviyo/util.js | 56 ++++++++++++++++++---- src/v0/destinations/klaviyo/util.test.js | 39 ++++++++++++++- 3 files changed, 92 insertions(+), 13 deletions(-) diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index 08ac11d33c..8314387dba 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -6,7 +6,7 @@ const { ConfigurationError, InstrumentationError } = require('@rudderstack/integ const { EventType } = require('../../../constants'); const { CONFIG_CATEGORIES, MAPPING_CONFIG } = require('./config'); const { - batchEvents, + batchSubscriptionRequestV2, constructProfile, subscribeUserToListV2, buildRequest, @@ -133,7 +133,7 @@ const processEvent = (event) => { } return response; }; - +// {subscription:{}, event:{}, profile:{}} const processV2 = (event) => { const response = processEvent(event); const { destination } = event; @@ -199,7 +199,11 @@ const processRouterDestV2 = (inputs, reqMetadata) => { batchErrorRespList.push(errRespEvent); } }); - const batchedResponseList = batchEvents(subscribeRespList, profileRespList, destination); + const batchedResponseList = batchSubscriptionRequestV2( + subscribeRespList, + profileRespList, + destination, + ); const { anonymousTracking, identifiedTracking } = getTrackRequests(eventRespList, destination); // We are doing to maintain event ordering basically once a user is identified klaviyo does not allow user tracking based upon anonymous_id only diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index b24f8aa14b..4df3e125ec 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -268,6 +268,17 @@ const groupSubscribeResponsesUsingListId = (subscribeResponseList) => { ); return subscribeEventGroups; }; + +/** + * This function groups the subscription responses on list id + * @param {*} subscribeResponseList + * @returns + * Example subsribeResponseList = + * [ + * { payload: {id:'list_id', profile: {}}, metadata:{} }, + * { payload: {id:'list_id', profile: {}}, metadata:{} } + * ] + */ const groupSubscribeResponsesUsingListIdV2 = (subscribeResponseList) => { const subscribeEventGroups = lodash.groupBy( subscribeResponseList, @@ -515,12 +526,12 @@ const subscribeUserToListV2 = (message, traitsInfo, destination) => { /** * This Create a subscription payload to subscribe profile(s) to list listId * @param {*} listId - * @param {*} profile + * @param {*} profiles */ -const getSubscriptionPayload = (listId, profile) => ({ +const getSubscriptionPayload = (listId, profiles) => ({ data: { type: 'profile-subscription-bulk-create-job', - attributes: { profiles: { data: profile } }, + attributes: { profiles: { data: profiles } }, relationships: { list: { data: { @@ -578,6 +589,16 @@ const generateBatchedSubscriptionRequest = (events, destination) => { * @param {*} profileReq array of profile requests * @param {*} metadataArray array of metadata * @param {*} batchEventResponse + * Example: /** + * + * @param {*} subscribeEventGroups + * @param {*} identifyResponseList + * @returns + * Example: + * profileReq = [ + * { payload: {}, metadata:{} }, + * { payload: {}, metadata:{} } + * ] */ const updateBatchEventResponseWithProfileRequests = ( profileReq, @@ -593,6 +614,7 @@ const updateBatchEventResponseWithProfileRequests = ( ); } }); + // we are keeping profiles request prior to subscription ones batchEventResponse.batchedRequest.unshift(...profilesRequests); }; @@ -614,15 +636,26 @@ const getRemainingProfiles = (profileReq, subscriptionMetadataArray) => { * @param {*} subscribeRespList * @param {*} profileRespList * @param {*} eventRespList + * subscribeRespList = [ + * { payload: {id:'list_id', profile: {}}, metadata:{} }, + * { payload: {id:'list_id', profile: {}}, metadata:{} } + * ] + * profileRespList = [ + * { payload: {}, metadata:{} }, + * { payload: {}, metadata:{} } + * ] + * */ -const batchEvents = (subscribeRespList, profileRespList, destination) => { +const batchSubscriptionRequestV2 = (subscribeRespList, profileRespList, destination) => { const batchedResponseList = []; let remainingProfileReq = profileRespList; + const subscriptionMetadataArrayForAll = []; const subscribeEventGroups = groupSubscribeResponsesUsingListIdV2(subscribeRespList); Object.keys(subscribeEventGroups).forEach((listId) => { // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] const eventChunks = lodash.chunk(subscribeEventGroups[listId], MAX_BATCH_SIZE); - const batchedResponse = eventChunks.map((chunk) => { + const batchedResponse = []; + eventChunks.forEach((chunk) => { const batchEventResponse = generateBatchedSubscriptionRequest(chunk, destination); const { metadata: subscriptionMetadataArray, batchedRequest } = batchEventResponse; updateBatchEventResponseWithProfileRequests( @@ -630,12 +663,16 @@ const batchEvents = (subscribeRespList, profileRespList, destination) => { subscriptionMetadataArray, batchEventResponse, ); - remainingProfileReq = getRemainingProfiles(remainingProfileReq, subscriptionMetadataArray); - return getSuccessRespEvents(batchedRequest, subscriptionMetadataArray, destination, true); + subscriptionMetadataArrayForAll.push(...subscriptionMetadataArray); + batchedResponse.push( + getSuccessRespEvents(batchedRequest, subscriptionMetadataArray, destination, true), + ); }); batchedResponseList.push(...batchedResponse); }); const profiles = []; + remainingProfileReq = getRemainingProfiles(remainingProfileReq, subscriptionMetadataArrayForAll); + // push profiles for which there is no subscription remainingProfileReq.forEach((input) => { profiles.push( @@ -675,7 +712,7 @@ const getTrackRequests = (eventRespList, destination) => { const identifiedTracking = []; eventRespList.forEach((resp) => { const { payload, metadata } = resp; - const { attributes: profileAttributes } = payload.data.attributes.profile.attributes; + const { attributes: profileAttributes } = payload.data.attributes.profile.data; // eslint-disable-next-line @typescript-eslint/naming-convention const { email, phone_number, external_id } = profileAttributes; const request = getSuccessRespEvents( @@ -702,8 +739,9 @@ module.exports = { constructProfile, subscribeUserToListV2, getProfileMetadataAndMetadataFields, - batchEvents, + batchSubscriptionRequestV2, buildRequest, buildSubscriptionRequest, getTrackRequests, + groupSubscribeResponsesUsingListIdV2, }; diff --git a/src/v0/destinations/klaviyo/util.test.js b/src/v0/destinations/klaviyo/util.test.js index 36749e2311..ddf08a924c 100644 --- a/src/v0/destinations/klaviyo/util.test.js +++ b/src/v0/destinations/klaviyo/util.test.js @@ -1,4 +1,7 @@ -const { getProfileMetadataAndMetadataFields } = require('./util'); +const { + getProfileMetadataAndMetadataFields, + groupSubscribeResponsesUsingListIdV2, +} = require('./util'); describe('getProfileMetadataAndMetadataFields', () => { // Correctly generates metadata with fields to unset, append, and unappend when all fields are provided @@ -68,3 +71,37 @@ describe('getProfileMetadataAndMetadataFields', () => { expect(result).toEqual({ meta: { patch_properties: {} }, metadataFields: [] }); }); }); + +describe('groupSubscribeResponsesUsingListIdV2', () => { + // Groups subscription responses by listId correctly + it('should group subscription responses by listId correctly when given a valid list', () => { + const subscribeResponseList = [ + { payload: { listId: 'list_1', profile: {} }, metadata: {} }, + { payload: { listId: 'list_1', profile: {} }, metadata: {} }, + { payload: { listId: 'list_2', profile: {} }, metadata: {} }, + ]; + + const expectedOutput = { + list_1: [ + { payload: { listId: 'list_1', profile: {} }, metadata: {} }, + { payload: { listId: 'list_1', profile: {} }, metadata: {} }, + ], + list_2: [{ payload: { listId: 'list_2', profile: {} }, metadata: {} }], + }; + + const result = groupSubscribeResponsesUsingListIdV2(subscribeResponseList); + + expect(result).toEqual(expectedOutput); + }); + + // Handles empty subscription response list + it('should return an empty object when given an empty subscription response list', () => { + const subscribeResponseList = []; + + const expectedOutput = {}; + + const result = groupSubscribeResponsesUsingListIdV2(subscribeResponseList); + + expect(result).toEqual(expectedOutput); + }); +}); From c6715184188ab4066f68e307a0f9a41d12bcc245 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Wed, 17 Jul 2024 15:57:14 +0530 Subject: [PATCH 12/26] chore: small fix for track call and add util test --- src/v0/destinations/klaviyo/util.js | 2 +- .../destinations/klaviyo/processor/trackTestDataV2.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 4df3e125ec..57820e9631 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -447,7 +447,7 @@ const constructProfile = (message, destination, isIdentifyCall) => { customPropertyPayload = flattenProperties ? flattenJson(customPropertyPayload, '.', 'normal', false) : customPropertyPayload; - if (isIdentifyCall && enforceEmailAsPrimary) { + if (enforceEmailAsPrimary) { if (!profileAttributes.email && !profileAttributes.phone_number && !profileId) { throw new InstrumentationError('None of email and phone are present in the payload'); } diff --git a/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts index f9e5a0233e..5af7e8fb40 100644 --- a/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts @@ -47,7 +47,6 @@ const commonOutputHeaders = { const profileAttributes = { email: 'test@rudderstack.com', phone_number: '9112340375', - external_id: 'sajal12', anonymous_id: '9c6bd77ea9da3e68', properties: { age: '22', @@ -128,7 +127,7 @@ export const trackTestData: ProcessorTestData[] = [ profile: { data: { type: 'profile', - attributes: profileAttributes, + attributes: { ...profileAttributes, external_id: 'sajal12' }, }, }, properties: commonProps, @@ -151,7 +150,7 @@ export const trackTestData: ProcessorTestData[] = [ id: 'klaviyo-track-150624-test-2', name: 'klaviyo', description: - "150624 -> Track event call, with make email or phone as primary identifier toggle on but it doesn't matter as this toggle is only for identify call", + '150624 -> Track event call, with make email or phone as primary identifier toggle on', scenario: 'Business', successCriteria: 'Response should contain only event payload with profile object and status code should be 200, for the event payload should contain contextual traits and properties in the payload, and email should be mapped to email and userId should be mapped to external_id as usual', @@ -213,6 +212,7 @@ export const trackTestData: ProcessorTestData[] = [ type: 'profile', attributes: { ...profileAttributes, + properties: { ...profileAttributes.properties, _id: 'sajal12' }, }, }, }, From 29db8d013a9bb876e986826e0227cd09aba8ae29 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Wed, 17 Jul 2024 16:27:31 +0530 Subject: [PATCH 13/26] fix: remove error in case no identifier is present --- src/v0/destinations/klaviyo/util.js | 3 - .../klaviyo/processor/identifyTestDataV2.ts | 57 ------------------- 2 files changed, 60 deletions(-) diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 57820e9631..c69e111425 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -448,9 +448,6 @@ const constructProfile = (message, destination, isIdentifyCall) => { ? flattenJson(customPropertyPayload, '.', 'normal', false) : customPropertyPayload; if (enforceEmailAsPrimary) { - if (!profileAttributes.email && !profileAttributes.phone_number && !profileId) { - throw new InstrumentationError('None of email and phone are present in the payload'); - } delete profileAttributes.external_id; // so that multiple profiles are not found, one w.r.t email and one for external_id customPropertyPayload = { ...customPropertyPayload, diff --git a/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts index 830dffec47..3f0bfda4db 100644 --- a/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts @@ -291,63 +291,6 @@ export const identifyData: ProcessorTestData[] = [ }, }, }, - { - id: 'klaviyo-identify-150624-test-3', - name: 'klaviyo', - description: - '150624 -> Identify call without email and phone & enforceEmailAsPrimary enabled from UI', - scenario: 'Business', - successCriteria: - 'Response should contain error message and status code should be 400, as we are not sending email and phone in the payload and enforceEmailAsPrimary is enabled from UI', - feature: 'processor', - module: 'destination', - version: 'v0', - input: { - request: { - body: [ - { - destination: overrideDestination(destination, { enforceEmailAsPrimary: true }), - message: generateSimplifiedIdentifyPayload({ - sentAt, - userId, - context: { - traits: removeUndefinedAndNullValues({ - ...commonTraits, - email: undefined, - phone: undefined, - }), - }, - anonymousId, - originalTimestamp, - }), - metadata: generateMetadata(7), - }, - ], - }, - }, - output: { - response: { - status: 200, - body: [ - { - error: 'None of email and phone are present in the payload', - statTags: { - destType: 'KLAVIYO', - errorCategory: 'dataValidation', - errorType: 'instrumentation', - feature: 'processor', - implementation: 'native', - module: 'destination', - destinationId: 'default-destinationId', - workspaceId: 'default-workspaceId', - }, - statusCode: 400, - metadata: generateMetadata(7), - }, - ], - }, - }, - }, { id: 'klaviyo-identify-150624-test-5', name: 'klaviyo', From 0877ea313a290324540bc010224f9c6fc40671cd Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Wed, 17 Jul 2024 18:47:03 +0530 Subject: [PATCH 14/26] chore: remove env var lead version switching and add test cases --- src/v0/destinations/klaviyo/transform.js | 5 +- .../destinations/klaviyo/router/dataV2.ts | 507 +++++++++++++++++- 2 files changed, 508 insertions(+), 4 deletions(-) diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js index a63fa65bb8..97872fdc96 100644 --- a/src/v0/destinations/klaviyo/transform.js +++ b/src/v0/destinations/klaviyo/transform.js @@ -12,7 +12,6 @@ const { ecomEvents, eventNameMapping, jsonNameMapping, - useUpdatedKlaviyoAPI, } = require('./config'); const { processRouterDestV2, processV2 } = require('./transformV2'); const { @@ -279,7 +278,7 @@ const groupRequestHandler = (message, category, destination) => { // Main event processor using specific handler funcs const processEvent = async (event, reqMetadata) => { const { message, destination, metadata } = event; - if (destination.Config?.version === 'v2' || useUpdatedKlaviyoAPI) { + if (destination.Config?.version === 'v2') { return processV2(event, reqMetadata); } if (!message.type) { @@ -334,7 +333,7 @@ const getEventChunks = (event, subscribeRespList, nonSubscribeRespList) => { const processRouterDest = async (inputs, reqMetadata) => { const { destination } = inputs[0]; // This is used to switch to latest API version - if (destination.Config?.version === 'v2' || useUpdatedKlaviyoAPI) { + if (destination.Config?.version === 'v2') { return processRouterDestV2(inputs, reqMetadata); } let batchResponseList = []; diff --git a/test/integrations/destinations/klaviyo/router/dataV2.ts b/test/integrations/destinations/klaviyo/router/dataV2.ts index ff8f1679b2..a958328053 100644 --- a/test/integrations/destinations/klaviyo/router/dataV2.ts +++ b/test/integrations/destinations/klaviyo/router/dataV2.ts @@ -1,7 +1,8 @@ import { Destination } from '../../../../../src/types'; import { RouterTestData } from '../../../testTypes'; -import { generateMetadata } from '../../../testUtils'; import { routerRequestV2 } from './commonConfig'; +import { generateMetadata, generateSimplifiedTrackPayload } from '../../../testUtils'; + const destination: Destination = { ID: '123', Name: 'klaviyo', @@ -234,4 +235,508 @@ export const dataV2: RouterTestData[] = [ }, }, }, + { + id: 'klaviyo-router-150624-test-2', + name: 'klaviyo', + description: + '150624 -> Router tests to have some anonymous track event, some identify events with subscription and some identified track event', + scenario: 'Framework', + successCriteria: + 'All the subscription events should be batched and respective profile requests should also be placed in same batched request', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + // user 1 track call with userId and anonymousId + channel: 'web', + context: { + traits: { + email: 'testklaviyo1@email.com', + firstname: 'Test Klaviyo 1', + }, + }, + type: 'track', + anonymousId: 'anonTestKlaviyo1', + userId: 'testKlaviyo1', + event: 'purchase', + properties: { + price: '12', + }, + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + // Anonymous Tracking -> user 2 track call with anonymousId only + channel: 'web', + context: { + traits: {}, + }, + type: 'track', + anonymousId: 'anonTestKlaviyo2', + event: 'viewed product', + properties: { + price: '120', + }, + }, + metadata: generateMetadata(2), + destination, + }, + { + message: { + // user 2 idenitfy call with anonymousId and subscription + channel: 'web', + traits: { + email: 'testklaviyo2@rs.com', + firstname: 'Test Klaviyo 2', + properties: { + subscribe: true, + listId: 'configListId', + consent: ['email'], + }, + }, + context: {}, + anonymousId: 'anonTestKlaviyo2', + type: 'identify', + userId: 'testKlaviyo2', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(3), + destination, + }, + { + message: { + // user 2 track call with email only + channel: 'web', + context: { + traits: { + email: 'testklaviyo2@email.com', + firstname: 'Test Klaviyo 2', + }, + }, + type: 'track', + userId: 'testKlaviyo2', + event: 'purchase', + properties: { + price: '120', + }, + }, + metadata: generateMetadata(4), + destination, + }, + { + message: { + // for user 3 identify call without anonymousId and subscriptiontraits: + channel: 'web', + traits: { + email: 'testklaviyo3@rs.com', + firstname: 'Test Klaviyo 3', + properties: { + subscribe: true, + listId: 'configListId', + consent: ['email', 'sms'], + }, + }, + context: {}, + type: 'identify', + userId: 'testKlaviyo3', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(5), + destination, + }, + { + message: { + channel: 'web', + context: { + traits: { + email: 'testklaviyo3@email.com', + firstname: 'Test klaviyo3', + anonymousId: '1111', + }, + }, + type: 'identify', + anonymousId: '', + userId: 'testKlaviyo3', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + metadata: generateMetadata(6), + destination, + }, + ], + destType: 'klaviyo', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/events', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'event', + attributes: { + properties: { + price: '120', + }, + profile: { + data: { + type: 'profile', + attributes: { + anonymous_id: 'anonTestKlaviyo2', + properties: {}, + meta: { + patch_properties: {}, + }, + }, + }, + }, + metric: { + data: { + type: 'metric', + attributes: { + name: 'viewed product', + }, + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(2)], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-import', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo3', + email: 'testklaviyo3@email.com', + first_name: 'Test klaviyo3', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(6)], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-import', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo2', + anonymous_id: 'anonTestKlaviyo2', + email: 'testklaviyo2@rs.com', + first_name: 'Test Klaviyo 2', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-import', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo3', + email: 'testklaviyo3@rs.com', + first_name: 'Test Klaviyo 3', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + 'Content-Type': 'application/json', + Accept: 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'testklaviyo2@rs.com', + subscriptions: { + email: { + marketing: { + consent: 'SUBSCRIBED', + }, + }, + }, + }, + }, + { + type: 'profile', + attributes: { + email: 'testklaviyo3@rs.com', + subscriptions: { + email: { + marketing: { + consent: 'SUBSCRIBED', + }, + }, + }, + }, + }, + ], + }, + }, + relationships: { + list: { + data: { + type: 'list', + id: 'configListId', + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(3), generateMetadata(5)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/events', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'event', + attributes: { + properties: { + price: '12', + }, + profile: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo1', + anonymous_id: 'anonTestKlaviyo1', + email: 'testklaviyo1@email.com', + first_name: 'Test Klaviyo 1', + properties: {}, + meta: { + patch_properties: {}, + }, + }, + }, + }, + metric: { + data: { + type: 'metric', + attributes: { + name: 'purchase', + }, + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1)], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/events', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'event', + attributes: { + properties: { + price: '120', + }, + profile: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo2', + email: 'testklaviyo2@email.com', + first_name: 'Test Klaviyo 2', + properties: {}, + meta: { + patch_properties: {}, + }, + }, + }, + }, + metric: { + data: { + type: 'metric', + attributes: { + name: 'purchase', + }, + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(4)], + batched: false, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + }, ]; From d71c1ee7fe7a84415723ead3e8946e90717a7220 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Wed, 17 Jul 2024 19:08:17 +0530 Subject: [PATCH 15/26] chore: address comments --- src/v0/destinations/klaviyo/transform.js | 4 ++-- .../klaviyo/processor/groupTestDataV2.ts | 2 +- .../klaviyo/processor/identifyTestDataV2.ts | 2 +- .../klaviyo/processor/screenTestDataV2.ts | 2 +- .../klaviyo/processor/trackTestDataV2.ts | 2 +- .../klaviyo/processor/validationTestDataV2.ts | 2 +- .../destinations/klaviyo/router/commonConfig.ts | 14 +++++++------- .../destinations/klaviyo/router/dataV2.ts | 4 ++-- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js index 97872fdc96..b95fe0abf3 100644 --- a/src/v0/destinations/klaviyo/transform.js +++ b/src/v0/destinations/klaviyo/transform.js @@ -278,7 +278,7 @@ const groupRequestHandler = (message, category, destination) => { // Main event processor using specific handler funcs const processEvent = async (event, reqMetadata) => { const { message, destination, metadata } = event; - if (destination.Config?.version === 'v2') { + if (destination.Config?.apiVersion === 'v2') { return processV2(event, reqMetadata); } if (!message.type) { @@ -333,7 +333,7 @@ const getEventChunks = (event, subscribeRespList, nonSubscribeRespList) => { const processRouterDest = async (inputs, reqMetadata) => { const { destination } = inputs[0]; // This is used to switch to latest API version - if (destination.Config?.version === 'v2') { + if (destination.Config?.apiVersion === 'v2') { return processRouterDestV2(inputs, reqMetadata); } let batchResponseList = []; diff --git a/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts index 8a944b76df..bcb59f0dbb 100644 --- a/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts @@ -16,7 +16,7 @@ const destination: Destination = { Config: {}, }, Config: { - version: 'v2', + apiVersion: 'v2', privateApiKey: 'dummyPrivateApiKey', consent: ['email'], }, diff --git a/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts index 3f0bfda4db..80ae918af0 100644 --- a/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts @@ -18,7 +18,7 @@ const destination: Destination = { Config: {}, }, Config: { - version: 'v2', + apiVersion: 'v2', privateApiKey: 'dummyPrivateApiKey', }, Enabled: true, diff --git a/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts index 3ccafb39a6..5ccbed600c 100644 --- a/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts @@ -16,7 +16,7 @@ const destination: Destination = { Config: {}, }, Config: { - version: 'v2', + apiVersion: 'v2', privateApiKey: 'dummyPrivateApiKey', }, Enabled: true, diff --git a/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts index 5af7e8fb40..d81ee9a4a1 100644 --- a/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts @@ -19,7 +19,7 @@ const destination: Destination = { }, Config: { privateApiKey: 'dummyPrivateApiKey', - version: 'v2', + apiVersion: 'v2', }, Enabled: true, WorkspaceID: '123', diff --git a/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts index 613356c9b5..10e2d15db0 100644 --- a/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/validationTestDataV2.ts @@ -13,7 +13,7 @@ const destination: Destination = { }, Config: { privateApiKey: 'dummyPrivateApiKey', - version: 'v2', + apiVersion: 'v2', }, Enabled: true, WorkspaceID: '123', diff --git a/test/integrations/destinations/klaviyo/router/commonConfig.ts b/test/integrations/destinations/klaviyo/router/commonConfig.ts index 8d9e1c2d06..a1297635eb 100644 --- a/test/integrations/destinations/klaviyo/router/commonConfig.ts +++ b/test/integrations/destinations/klaviyo/router/commonConfig.ts @@ -27,16 +27,16 @@ const destinationV2: Destination = { }, Config: { privateApiKey: 'dummyPrivateApiKey', - version: 'v2', + apiVersion: 'v2', }, Enabled: true, WorkspaceID: '123', Transformations: [], }; -const getRequest = (version) => { +const getRequest = (apiVersion) => { return [ { - destination: version === 'v2' ? destinationV2 : destination, + destination: apiVersion === 'v2' ? destinationV2 : destination, metadata: generateMetadata(1), message: { type: 'identify', @@ -82,7 +82,7 @@ const getRequest = (version) => { }, }, { - destination: version === 'v2' ? destinationV2 : destination, + destination: apiVersion === 'v2' ? destinationV2 : destination, metadata: generateMetadata(2), message: { type: 'identify', @@ -128,7 +128,7 @@ const getRequest = (version) => { }, }, { - destination: version === 'v2' ? destinationV2 : destination, + destination: apiVersion === 'v2' ? destinationV2 : destination, metadata: generateMetadata(3), message: { userId: 'user123', @@ -148,7 +148,7 @@ const getRequest = (version) => { }, }, { - destination: version === 'v2' ? destinationV2 : destination, + destination: apiVersion === 'v2' ? destinationV2 : destination, metadata: generateMetadata(4), message: { userId: 'user123', @@ -168,7 +168,7 @@ const getRequest = (version) => { }, }, { - destination: version === 'v2' ? destinationV2 : destination, + destination: apiVersion === 'v2' ? destinationV2 : destination, metadata: generateMetadata(5), message: { userId: 'user123', diff --git a/test/integrations/destinations/klaviyo/router/dataV2.ts b/test/integrations/destinations/klaviyo/router/dataV2.ts index a958328053..858fded0d2 100644 --- a/test/integrations/destinations/klaviyo/router/dataV2.ts +++ b/test/integrations/destinations/klaviyo/router/dataV2.ts @@ -1,7 +1,7 @@ import { Destination } from '../../../../../src/types'; import { RouterTestData } from '../../../testTypes'; import { routerRequestV2 } from './commonConfig'; -import { generateMetadata, generateSimplifiedTrackPayload } from '../../../testUtils'; +import { generateMetadata } from '../../../testUtils'; const destination: Destination = { ID: '123', @@ -14,7 +14,7 @@ const destination: Destination = { }, Config: { privateApiKey: 'dummyPrivateApiKey', - version: 'v2', + apiVersion: 'v2', }, Enabled: true, WorkspaceID: '123', From d15011755160829c5985f2d27ebc0f24da386fd5 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Fri, 19 Jul 2024 10:26:24 +0530 Subject: [PATCH 16/26] feat: onboard new api for klaviyo 15-06-2024 --- src/v0/destinations/klaviyo/util.js | 16 +++++ .../klaviyo/processor/groupTestDataV2.ts | 11 ---- .../destinations/klaviyo/router/dataV2.ts | 60 ++++++++++++++++++- 3 files changed, 75 insertions(+), 12 deletions(-) diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index c69e111425..2eb83646b3 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -725,6 +725,21 @@ const getTrackRequests = (eventRespList, destination) => { }); return { anonymousTracking, identifiedTracking }; }; + +/** + * This function checks for the transformed event structure and accordingly send back the batched responses + * @param {*} event + */ +const fetchTransformedEvents = (event) => { + const { message, destination, metadata } = event; + // checking if we have any output field if yes then we return message.output + return getSuccessRespEvents( + message.output || message, + Array.isArray(metadata) ? metadata : [metadata], + destination, + ); +}; + module.exports = { subscribeUserToList, createCustomerProperties, @@ -741,4 +756,5 @@ module.exports = { buildSubscriptionRequest, getTrackRequests, groupSubscribeResponsesUsingListIdV2, + fetchTransformedEvents, }; diff --git a/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts index bcb59f0dbb..dcd7fbc38e 100644 --- a/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts @@ -109,17 +109,6 @@ export const groupTestData: ProcessorTestData[] = [ relationships: subscriptionRelations, }, }, - a: { - data: { - attributes: { - list_id: 'XUepkK', - subscriptions: [ - { email: 'test@rudderstack.com', phone_number: '+12 345 678 900' }, - ], - }, - type: 'profile-subscription-bulk-create-job', - }, - }, endpoint: endpoint, headers: headers, method: 'POST', diff --git a/test/integrations/destinations/klaviyo/router/dataV2.ts b/test/integrations/destinations/klaviyo/router/dataV2.ts index 858fded0d2..65f704ddf9 100644 --- a/test/integrations/destinations/klaviyo/router/dataV2.ts +++ b/test/integrations/destinations/klaviyo/router/dataV2.ts @@ -1,7 +1,7 @@ import { Destination } from '../../../../../src/types'; import { RouterTestData } from '../../../testTypes'; import { routerRequestV2 } from './commonConfig'; -import { generateMetadata } from '../../../testUtils'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; const destination: Destination = { ID: '123', @@ -37,6 +37,44 @@ const subscriptionRelations = { }, }; +const commonOutputSubscriptionProps = { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'test@rudderstack.com', + phone_number: '+12 345 678 900', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + ], + }, +}; + +const alreadyTransformedEvent = { + message: { + output: transformResultBuilder({ + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: commonOutputSubscriptionProps, + relationships: subscriptionRelations, + }, + }, + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers: headers, + method: 'POST', + userId: '', + }), + statusCode: 200, + }, + metadata: generateMetadata(10), + destination, +}; + export const dataV2: RouterTestData[] = [ { id: 'klaviyo-router-150624-test-1', @@ -250,6 +288,7 @@ export const dataV2: RouterTestData[] = [ request: { body: { input: [ + alreadyTransformedEvent, { message: { // user 1 track call with userId and anonymousId @@ -386,6 +425,25 @@ export const dataV2: RouterTestData[] = [ status: 200, body: { output: [ + { + batchedRequest: transformResultBuilder({ + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: commonOutputSubscriptionProps, + relationships: subscriptionRelations, + }, + }, + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers: headers, + method: 'POST', + userId: '', + }), + metadata: [generateMetadata(10)], + batched: false, + statusCode: 200, + destination, + }, { batchedRequest: { version: '1', From caab5847b85a2146306ee30b9053c6cf98105931 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Fri, 19 Jul 2024 10:54:53 +0530 Subject: [PATCH 17/26] chore: solve duplicate import --- src/v0/destinations/klaviyo/util.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index bd4c87c2f3..4dcfe01acf 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -1,11 +1,7 @@ const set = require('set-value'); const { defaultRequestConfig } = require('rudder-transformer-cdk/build/utils'); const lodash = require('lodash'); -const { - NetworkError, - InstrumentationError, - isDefinedAndNotNull, -} = require('@rudderstack/integrations-lib'); +const { NetworkError, InstrumentationError } = require('@rudderstack/integrations-lib'); const { WhiteListedTraits } = require('../../../constants'); const { constructPayload, From 11ccd2a914074836feed04eaa466383541113afd Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Fri, 19 Jul 2024 11:03:58 +0530 Subject: [PATCH 18/26] chore: resolve merge conflicts --- src/v0/destinations/klaviyo/transformV2.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index 8314387dba..da259bec6a 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -12,6 +12,7 @@ const { buildRequest, buildSubscriptionRequest, getTrackRequests, + fetchTransformedEvents, } = require('./util'); const { constructPayload, @@ -42,7 +43,6 @@ const identifyRequestHandler = (message, category, destination) => { if (traitsInfo?.properties?.subscribe && (traitsInfo.properties?.listId || listId)) { response.subscription = subscribeUserToListV2(message, traitsInfo, destination); } - // returning list if subscription to a list is to be done else returning an object to upsert profile return response; }; @@ -166,7 +166,7 @@ const getEventChunks = (input, subscribeRespList, profileRespList, eventRespList }; const processRouterDestV2 = (inputs, reqMetadata) => { - let batchResponseList = []; + const batchResponseList = []; const batchErrorRespList = []; const subscribeRespList = []; const profileRespList = []; @@ -176,12 +176,7 @@ const processRouterDestV2 = (inputs, reqMetadata) => { try { if (event.message.statusCode) { // already transformed event - getEventChunks( - { message: event.message, metadata: event.metadata, destination }, - subscribeRespList, - profileRespList, - eventRespList, - ); + batchResponseList.push(fetchTransformedEvents(event)); } else { // if not transformed getEventChunks( @@ -207,7 +202,7 @@ const processRouterDestV2 = (inputs, reqMetadata) => { const { anonymousTracking, identifiedTracking } = getTrackRequests(eventRespList, destination); // We are doing to maintain event ordering basically once a user is identified klaviyo does not allow user tracking based upon anonymous_id only - batchResponseList = [...anonymousTracking, ...batchedResponseList, ...identifiedTracking]; + batchResponseList.push(...anonymousTracking, ...batchedResponseList, ...identifiedTracking); return [...batchResponseList, ...batchErrorRespList]; }; From f1259fdbf25f21cdd683144047ded232aabbb574 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Fri, 19 Jul 2024 11:59:35 +0530 Subject: [PATCH 19/26] feat: add retl flow specific code --- src/v0/destinations/klaviyo/transformV2.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index da259bec6a..7659687d36 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -3,7 +3,7 @@ /* eslint-disable array-callback-return */ const get = require('get-value'); const { ConfigurationError, InstrumentationError } = require('@rudderstack/integrations-lib'); -const { EventType } = require('../../../constants'); +const { EventType, MappedToDestinationKey } = require('../../../constants'); const { CONFIG_CATEGORIES, MAPPING_CONFIG } = require('./config'); const { batchSubscriptionRequestV2, @@ -13,12 +13,15 @@ const { buildSubscriptionRequest, getTrackRequests, fetchTransformedEvents, + addSubcribeFlagToTraits, } = require('./util'); const { constructPayload, getFieldValueFromMessage, removeUndefinedAndNullValues, handleRtTfSingleEventError, + addExternalIdToTraits, + adduserIdFromExternalId, flattenJson, } = require('../../util'); @@ -38,7 +41,13 @@ const identifyRequestHandler = (message, category, destination) => { const { listId } = destination.Config; const payload = removeUndefinedAndNullValues(constructProfile(message, destination, true)); const response = { profile: payload }; - const traitsInfo = getFieldValueFromMessage(message, 'traits'); + let traitsInfo = getFieldValueFromMessage(message, 'traits'); + const mappedToDestination = get(message, MappedToDestinationKey); + if (mappedToDestination) { + addExternalIdToTraits(message); + adduserIdFromExternalId(message); + traitsInfo = addSubcribeFlagToTraits(traitsInfo); + } // check if user wants to subscribe profile or not and listId is present or not if (traitsInfo?.properties?.subscribe && (traitsInfo.properties?.listId || listId)) { response.subscription = subscribeUserToListV2(message, traitsInfo, destination); From 09e06ce5a63c77ed97bd2345589e6828294a9d05 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Fri, 19 Jul 2024 17:27:11 +0530 Subject: [PATCH 20/26] chore: address comments --- src/v0/destinations/klaviyo/batchUtil.js | 186 +++++++++++++++ src/v0/destinations/klaviyo/transformV2.js | 10 +- src/v0/destinations/klaviyo/util.js | 164 +------------ src/v0/destinations/klaviyo/util.test.js | 6 +- .../destinations/klaviyo/router/dataV2.ts | 221 ++++++++++++++++++ 5 files changed, 415 insertions(+), 172 deletions(-) create mode 100644 src/v0/destinations/klaviyo/batchUtil.js diff --git a/src/v0/destinations/klaviyo/batchUtil.js b/src/v0/destinations/klaviyo/batchUtil.js new file mode 100644 index 0000000000..90b36de513 --- /dev/null +++ b/src/v0/destinations/klaviyo/batchUtil.js @@ -0,0 +1,186 @@ +const lodash = require('lodash'); +const { defaultBatchRequestConfig, getSuccessRespEvents } = require('../../util'); +const { JSON_MIME_TYPE } = require('../../util/constant'); +const { BASE_ENDPOINT, CONFIG_CATEGORIES, MAX_BATCH_SIZE, revision } = require('./config'); +const { buildRequest, getSubscriptionPayload } = require('./util'); +/** + * This function groups the subscription responses on list id + * @param {*} subscribeResponseList + * @returns + * Example subsribeResponseList = + * [ + * { payload: {id:'list_id', profile: {}}, metadata:{} }, + * { payload: {id:'list_id', profile: {}}, metadata:{} } + * ] + */ +const groupSubscribeResponsesUsingListIdV2 = (subscribeResponseList) => { + const subscribeEventGroups = lodash.groupBy( + subscribeResponseList, + (event) => event.payload.listId, + ); + return subscribeEventGroups; +}; + +/** + * This function returns the list of profileReq which do not metadata common with subcriptionMetadataArray + * @param {*} profileReq + * @param {*} subscriptionMetadataArray + * @returns + */ +const getRemainingProfiles = (profileReq, subscriptionMetadataArray) => { + const subscriptionListJobIds = subscriptionMetadataArray.map((metadata) => metadata.jobId); + return profileReq.filter((profile) => !subscriptionListJobIds.includes(profile.metadata.jobId)); +}; + +/** + * This function builds all the profile requests whose metadata is not there in subscriptionMetadataArray + * @param {*} profileRespList + * @param {*} subscriptionMetadataArray + * @param {*} destination + * @returns + */ +const getProfiles = (profileRespList, subscriptionMetadataArray, destination) => { + const profiles = []; + const remainingProfileReq = getRemainingProfiles(profileRespList, subscriptionMetadataArray); + remainingProfileReq.forEach((input) => { + profiles.push( + getSuccessRespEvents( + buildRequest(input.payload, destination, CONFIG_CATEGORIES.IDENTIFYV2), + [input.metadata], + destination, + ), + ); + }); + return profiles; +}; + +/** + * This function takes susbscriptions as input and batches them into a single request body + * @param {events} + * events= [ + * { payload: {id:'list_id', profile: {}}, metadata:{} }, + * { payload: {id:'list_id', profile: {}}, metadata:{} } + * ] + */ +const generateBatchedSubscriptionRequest = (events, destination) => { + const batchEventResponse = defaultBatchRequestConfig(); + const metadata = []; + // fetching listId from first event as listId is same for all the events + const listId = events[0].payload?.listId; + const profiles = []; // list of profiles to be subscribes + // Batch profiles into dest batch structure + events.forEach((ev) => { + profiles.push(...ev.payload.profile); + metadata.push(ev.metadata); + }); + + batchEventResponse.batchedRequest = Object.values(batchEventResponse); + batchEventResponse.batchedRequest[0].body.JSON = getSubscriptionPayload(listId, profiles); + + batchEventResponse.batchedRequest[0].endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; + + batchEventResponse.batchedRequest[0].headers = { + Authorization: `Klaviyo-API-Key ${destination.Config.privateApiKey}`, + 'Content-Type': JSON_MIME_TYPE, + Accept: JSON_MIME_TYPE, + revision, + }; + + return { + ...batchEventResponse, + metadata, + destination, + }; +}; + +/** + * This function fetches the profileRequests with metadata present in metadata array build a request for them + * and add these requests batchEvent Response + * @param {*} profileReq array of profile requests + * @param {*} metadataArray array of metadata + * @param {*} batchEventResponse + * Example: /** + * + * @param {*} subscribeEventGroups + * @param {*} identifyResponseList + * @returns + * Example: + * profileReq = [ + * { payload: {}, metadata:{} }, + * { payload: {}, metadata:{} } + * ] + */ +const updateBatchEventResponseWithProfileRequests = ( + profileReqArr, + subscriptionMetadataArray, + batchEventResponse, +) => { + const subscriptionListJobIds = subscriptionMetadataArray.map((metadata) => metadata.jobId); + const profilesRequests = []; + profileReqArr.forEach((profile) => { + if (subscriptionListJobIds.includes(profile.metadata.jobId)) { + profilesRequests.push( + buildRequest(profile.payload, batchEventResponse.destination, CONFIG_CATEGORIES.IDENTIFYV2), + ); + } + }); + // we are keeping profiles request prior to subscription ones + batchEventResponse.batchedRequest.unshift(...profilesRequests); +}; + +const processSubscribeChunk = (chunk, destination, profileRespList) => { + const batchEventResponse = generateBatchedSubscriptionRequest(chunk, destination); + const { metadata: subscriptionMetadataArray } = batchEventResponse; + updateBatchEventResponseWithProfileRequests( + profileRespList, + subscriptionMetadataArray, + batchEventResponse, + ); + return batchEventResponse; +}; + +/** + * This function batches the requests. Alogorithm + * Batch events from Subscribe Resp List having same listId/groupId to be subscribed and have their metadata array + * For this metadata array get all profileRequests and add them prior to batched Subscribe Request in the same batched Request + * Make another batched request for the remaning profile requests and another for all the event requests + * @param {*} subscribeRespList + * @param {*} profileRespList + * @param {*} eventRespList + * subscribeRespList = [ + * { payload: {id:'list_id', profile: {}}, metadata:{} }, + * { payload: {id:'list_id', profile: {}}, metadata:{} } + * ] + * profileRespList = [ + * { payload: {}, metadata:{} }, + * { payload: {}, metadata:{} } + * ] + * + */ +const batchSubscriptionRequestV2 = (subscribeRespList, profileRespList, destination) => { + const batchedResponseList = []; + const subscriptionMetadataArrayForAll = []; + const subscribeEventGroups = groupSubscribeResponsesUsingListIdV2(subscribeRespList); + Object.keys(subscribeEventGroups).forEach((listId) => { + // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] + const eventChunks = lodash.chunk(subscribeEventGroups[listId], MAX_BATCH_SIZE); + const batchedResponse = []; + eventChunks.forEach((chunk) => { + // returns subscriptionMetadata and batchEventResponse + const { metadata: subscriptionMetadataArray, batchedRequest } = processSubscribeChunk( + chunk, + destination, + profileRespList, + ); + subscriptionMetadataArrayForAll.push(...subscriptionMetadataArray); + batchedResponse.push( + getSuccessRespEvents(batchedRequest, subscriptionMetadataArray, destination, true), + ); + }); + batchedResponseList.push(...batchedResponse); + }); + const profiles = getProfiles(profileRespList, subscriptionMetadataArrayForAll, destination); + + return [...profiles, ...batchedResponseList]; +}; +module.exports = { batchSubscriptionRequestV2, groupSubscribeResponsesUsingListIdV2 }; diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index 7659687d36..2335e9339d 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -6,15 +6,15 @@ const { ConfigurationError, InstrumentationError } = require('@rudderstack/integ const { EventType, MappedToDestinationKey } = require('../../../constants'); const { CONFIG_CATEGORIES, MAPPING_CONFIG } = require('./config'); const { - batchSubscriptionRequestV2, constructProfile, subscribeUserToListV2, buildRequest, buildSubscriptionRequest, getTrackRequests, fetchTransformedEvents, - addSubcribeFlagToTraits, + addSubscribeFlagToTraits, } = require('./util'); +const { batchSubscriptionRequestV2 } = require('./batchUtil'); const { constructPayload, getFieldValueFromMessage, @@ -39,15 +39,15 @@ const { const identifyRequestHandler = (message, category, destination) => { // If listId property is present try to subscribe/member user in list const { listId } = destination.Config; - const payload = removeUndefinedAndNullValues(constructProfile(message, destination, true)); - const response = { profile: payload }; let traitsInfo = getFieldValueFromMessage(message, 'traits'); const mappedToDestination = get(message, MappedToDestinationKey); if (mappedToDestination) { addExternalIdToTraits(message); adduserIdFromExternalId(message); - traitsInfo = addSubcribeFlagToTraits(traitsInfo); + traitsInfo = addSubscribeFlagToTraits(traitsInfo); } + const payload = removeUndefinedAndNullValues(constructProfile(message, destination, true)); + const response = { profile: payload }; // check if user wants to subscribe profile or not and listId is present or not if (traitsInfo?.properties?.subscribe && (traitsInfo.properties?.listId || listId)) { response.subscription = subscribeUserToListV2(message, traitsInfo, destination); diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 4dcfe01acf..1e68aa2e09 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -269,24 +269,6 @@ const groupSubscribeResponsesUsingListId = (subscribeResponseList) => { return subscribeEventGroups; }; -/** - * This function groups the subscription responses on list id - * @param {*} subscribeResponseList - * @returns - * Example subsribeResponseList = - * [ - * { payload: {id:'list_id', profile: {}}, metadata:{} }, - * { payload: {id:'list_id', profile: {}}, metadata:{} } - * ] - */ -const groupSubscribeResponsesUsingListIdV2 = (subscribeResponseList) => { - const subscribeEventGroups = lodash.groupBy( - subscribeResponseList, - (event) => event.payload.listId, - ); - return subscribeEventGroups; -}; - const getBatchedResponseList = (subscribeEventGroups, identifyResponseList) => { let batchedResponseList = []; Object.keys(subscribeEventGroups).forEach((listId) => { @@ -540,149 +522,6 @@ const getSubscriptionPayload = (listId, profiles) => ({ }, }); -/** - * This function takes susbscriptions as input and batches them into a single request body - * @param {events} - * events= [ - * { payload: {id:'list_id', profile: {}}, metadata:{} }, - * { payload: {id:'list_id', profile: {}}, metadata:{} } - * ] - */ - -const generateBatchedSubscriptionRequest = (events, destination) => { - const batchEventResponse = defaultBatchRequestConfig(); - const metadata = []; - // fetching listId from first event as listId is same for all the events - const listId = events[0].payload?.listId; - const profiles = []; // list of profiles to be subscribes - // Batch profiles into dest batch structure - events.forEach((ev) => { - profiles.push(...ev.payload.profile); - metadata.push(ev.metadata); - }); - - batchEventResponse.batchedRequest = Object.values(batchEventResponse); - batchEventResponse.batchedRequest[0].body.JSON = getSubscriptionPayload(listId, profiles); - - batchEventResponse.batchedRequest[0].endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; - - batchEventResponse.batchedRequest[0].headers = { - Authorization: `Klaviyo-API-Key ${destination.Config.privateApiKey}`, - 'Content-Type': JSON_MIME_TYPE, - Accept: JSON_MIME_TYPE, - revision, - }; - - return { - ...batchEventResponse, - metadata, - destination, - }; -}; - -/** - * This function fetches the profileRequests with metadata present in metadata array build a request for them - * and add these requests batchEvent Response - * @param {*} profileReq array of profile requests - * @param {*} metadataArray array of metadata - * @param {*} batchEventResponse - * Example: /** - * - * @param {*} subscribeEventGroups - * @param {*} identifyResponseList - * @returns - * Example: - * profileReq = [ - * { payload: {}, metadata:{} }, - * { payload: {}, metadata:{} } - * ] - */ -const updateBatchEventResponseWithProfileRequests = ( - profileReq, - subscriptionMetadataArray, - batchEventResponse, -) => { - const subscriptionListJobIds = subscriptionMetadataArray.map((metadata) => metadata.jobId); - const profilesRequests = []; - profileReq.forEach((profile) => { - if (subscriptionListJobIds.includes(profile.metadata.jobId)) { - profilesRequests.push( - buildRequest(profile.payload, batchEventResponse.destination, CONFIG_CATEGORIES.IDENTIFYV2), - ); - } - }); - // we are keeping profiles request prior to subscription ones - batchEventResponse.batchedRequest.unshift(...profilesRequests); -}; - -/** - * This function returns the list of profileReq which do not metadata common with subcriptionMetadataArray - * @param {*} profileReq - * @param {*} subscriptionMetadataArray - * @returns - */ -const getRemainingProfiles = (profileReq, subscriptionMetadataArray) => { - const subscriptionListJobIds = subscriptionMetadataArray.map((metadata) => metadata.jobId); - return profileReq.filter((profile) => !subscriptionListJobIds.includes(profile.metadata.jobId)); -}; -/** - * This function batches the requests. Alogorithm - * Batch events from Subscribe Resp List having same listId/groupId to be subscribed and have their metadata array - * For this metadata array get all profileRequests and add them prior to batched Subscribe Request in the same batched Request - * Make another batched request for the remaning profile requests and another for all the event requests - * @param {*} subscribeRespList - * @param {*} profileRespList - * @param {*} eventRespList - * subscribeRespList = [ - * { payload: {id:'list_id', profile: {}}, metadata:{} }, - * { payload: {id:'list_id', profile: {}}, metadata:{} } - * ] - * profileRespList = [ - * { payload: {}, metadata:{} }, - * { payload: {}, metadata:{} } - * ] - * - */ -const batchSubscriptionRequestV2 = (subscribeRespList, profileRespList, destination) => { - const batchedResponseList = []; - let remainingProfileReq = profileRespList; - const subscriptionMetadataArrayForAll = []; - const subscribeEventGroups = groupSubscribeResponsesUsingListIdV2(subscribeRespList); - Object.keys(subscribeEventGroups).forEach((listId) => { - // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] - const eventChunks = lodash.chunk(subscribeEventGroups[listId], MAX_BATCH_SIZE); - const batchedResponse = []; - eventChunks.forEach((chunk) => { - const batchEventResponse = generateBatchedSubscriptionRequest(chunk, destination); - const { metadata: subscriptionMetadataArray, batchedRequest } = batchEventResponse; - updateBatchEventResponseWithProfileRequests( - remainingProfileReq, - subscriptionMetadataArray, - batchEventResponse, - ); - subscriptionMetadataArrayForAll.push(...subscriptionMetadataArray); - batchedResponse.push( - getSuccessRespEvents(batchedRequest, subscriptionMetadataArray, destination, true), - ); - }); - batchedResponseList.push(...batchedResponse); - }); - const profiles = []; - remainingProfileReq = getRemainingProfiles(remainingProfileReq, subscriptionMetadataArrayForAll); - - // push profiles for which there is no subscription - remainingProfileReq.forEach((input) => { - profiles.push( - getSuccessRespEvents( - buildRequest(input.payload, destination, CONFIG_CATEGORIES.IDENTIFYV2), - [input.metadata], - destination, - ), - ); - }); - return [...profiles, ...batchedResponseList]; -}; - /** * This function accepts subscriptions object and builds a request for it * @param {*} subscription @@ -773,11 +612,10 @@ module.exports = { constructProfile, subscribeUserToListV2, getProfileMetadataAndMetadataFields, - batchSubscriptionRequestV2, buildRequest, buildSubscriptionRequest, getTrackRequests, - groupSubscribeResponsesUsingListIdV2, fetchTransformedEvents, addSubscribeFlagToTraits, + getSubscriptionPayload, }; diff --git a/src/v0/destinations/klaviyo/util.test.js b/src/v0/destinations/klaviyo/util.test.js index ddf08a924c..755c937cff 100644 --- a/src/v0/destinations/klaviyo/util.test.js +++ b/src/v0/destinations/klaviyo/util.test.js @@ -1,8 +1,6 @@ -const { - getProfileMetadataAndMetadataFields, - groupSubscribeResponsesUsingListIdV2, -} = require('./util'); +const { getProfileMetadataAndMetadataFields } = require('./util'); +const { groupSubscribeResponsesUsingListIdV2 } = require('./batchUtil'); describe('getProfileMetadataAndMetadataFields', () => { // Correctly generates metadata with fields to unset, append, and unappend when all fields are provided it('should generate metadata with fields to unset, append, and unappend when all fields are provided', () => { diff --git a/test/integrations/destinations/klaviyo/router/dataV2.ts b/test/integrations/destinations/klaviyo/router/dataV2.ts index 65f704ddf9..2c43f5facf 100644 --- a/test/integrations/destinations/klaviyo/router/dataV2.ts +++ b/test/integrations/destinations/klaviyo/router/dataV2.ts @@ -797,4 +797,225 @@ export const dataV2: RouterTestData[] = [ }, }, }, + { + id: 'klaviyo-router-150624-test-3', + name: 'klaviyo', + description: '150624 -> Retl Router tests to have retl ', + scenario: 'Framework', + successCriteria: + 'All the subscription events should be batched and respective profile requests should also be placed in same batched request', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + channel: 'web', + context: { + mappedToDestination: 'true', + externalId: [ + { + id: 'testklaviyo1@email.com', + identifierType: 'email', + type: 'KLAVIYO-profile_v2', + }, + ], + traits: { + properties: { + subscribe: false, + }, + email: 'testklaviyo1@email.com', + firstname: 'Test Klaviyo 1', + }, + }, + type: 'identify', + anonymousId: 'anonTestKlaviyo1', + userId: 'testKlaviyo1', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + channel: 'web', + traits: { + email: 'testklaviyo2@rs.com', + firstname: 'Test Klaviyo 2', + properties: { + subscribe: true, + listId: 'configListId', + consent: ['email'], + }, + }, + context: { + mappedToDestination: 'true', + externalId: [ + { + id: 'testklaviyo2@rs.com', + identifierType: 'email', + type: 'KLAVIYO-profile_v2', + }, + ], + }, + anonymousId: 'anonTestKlaviyo2', + type: 'identify', + userId: 'testKlaviyo2', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(2), + destination, + }, + ], + destType: 'klaviyo', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-import', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testklaviyo1@email.com', + anonymous_id: 'anonTestKlaviyo1', + email: 'testklaviyo1@email.com', + first_name: 'Test Klaviyo 1', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1)], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-import', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testklaviyo2@rs.com', + anonymous_id: 'anonTestKlaviyo2', + email: 'testklaviyo2@rs.com', + first_name: 'Test Klaviyo 2', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + 'Content-Type': 'application/json', + Accept: 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'testklaviyo2@rs.com', + subscriptions: { + email: { + marketing: { + consent: 'SUBSCRIBED', + }, + }, + }, + }, + }, + ], + }, + }, + relationships: { + list: { + data: { + type: 'list', + id: 'configListId', + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(2)], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + }, ]; From e067fffbfec4b3413ba2f1735be4d4e33501a4eb Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 23 Jul 2024 12:10:42 +0530 Subject: [PATCH 21/26] feat: introduce job ordering in klaviyo --- src/v0/destinations/klaviyo/transformV2.js | 28 +- src/v0/destinations/klaviyo/util.js | 30 +- .../destinations/klaviyo/router/dataV2.ts | 836 +++++++++++------- test/integrations/testUtils.ts | 4 +- 4 files changed, 574 insertions(+), 324 deletions(-) diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index 2335e9339d..5982729315 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -22,6 +22,7 @@ const { handleRtTfSingleEventError, addExternalIdToTraits, adduserIdFromExternalId, + groupEventsByType, flattenJson, } = require('../../util'); @@ -174,7 +175,7 @@ const getEventChunks = (input, subscribeRespList, profileRespList, eventRespList } }; -const processRouterDestV2 = (inputs, reqMetadata) => { +const processRouter = (inputs, reqMetadata) => { const batchResponseList = []; const batchErrorRespList = []; const subscribeRespList = []; @@ -208,12 +209,31 @@ const processRouterDestV2 = (inputs, reqMetadata) => { profileRespList, destination, ); - const { anonymousTracking, identifiedTracking } = getTrackRequests(eventRespList, destination); + const trackRespList = getTrackRequests(eventRespList, destination); // We are doing to maintain event ordering basically once a user is identified klaviyo does not allow user tracking based upon anonymous_id only - batchResponseList.push(...anonymousTracking, ...batchedResponseList, ...identifiedTracking); + batchResponseList.push(...trackRespList, ...batchedResponseList); + + return { successEvents: batchResponseList, errorEvents: batchErrorRespList }; +}; - return [...batchResponseList, ...batchErrorRespList]; +const processRouterDestV2 = (inputs, reqMetadata) => { + /** + We are doing this to maintain the order of events not only fo transformation but for delivery as well + Job Id: 1 2 3 4 5 6 + Input : ['user1 track1', 'user1 identify 1', 'user1 track 2', 'user2 identify 1', 'user2 track 1', 'user1 track 3'] + Output after batching : [['user1 track1'],['user1 identify 1', 'user2 identify 1'], [ 'user1 track 2', 'user2 track 1', 'user1 track 3']] + Output after transformation: [1, [2,4], [3,5,6]] + */ + const inputsGroupedByType = groupEventsByType(inputs); + const respList = []; + const errList = []; + inputsGroupedByType.forEach((typedEventList) => { + const { successEvents, errorEvents } = processRouter(typedEventList, reqMetadata); + respList.push(...successEvents); + errList.push(...errorEvents); + }); + return [...respList, ...errList]; }; module.exports = { processV2, processRouterDestV2 }; diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 1e68aa2e09..76b51e4314 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -544,25 +544,17 @@ const buildSubscriptionRequest = (subscription, destination) => { const getTrackRequests = (eventRespList, destination) => { // building and pushing all the event requests - const anonymousTracking = []; - const identifiedTracking = []; - eventRespList.forEach((resp) => { - const { payload, metadata } = resp; - const { attributes: profileAttributes } = payload.data.attributes.profile.data; - // eslint-disable-next-line @typescript-eslint/naming-convention - const { email, phone_number, external_id } = profileAttributes; - const request = getSuccessRespEvents( - buildRequest(payload, destination, CONFIG_CATEGORIES.TRACKV2), - [metadata], - destination, - ); - if (email || phone_number || external_id) { - identifiedTracking.push(request); - } else { - anonymousTracking.push(request); - } - }); - return { anonymousTracking, identifiedTracking }; + const respList = []; + eventRespList.forEach((resp) => + respList.push( + getSuccessRespEvents( + buildRequest(resp.payload, destination, CONFIG_CATEGORIES.TRACKV2), + [resp.metadata], + destination, + ), + ), + ); + return respList; }; /** diff --git a/test/integrations/destinations/klaviyo/router/dataV2.ts b/test/integrations/destinations/klaviyo/router/dataV2.ts index 2c43f5facf..d385ec5d64 100644 --- a/test/integrations/destinations/klaviyo/router/dataV2.ts +++ b/test/integrations/destinations/klaviyo/router/dataV2.ts @@ -76,211 +76,506 @@ const alreadyTransformedEvent = { }; export const dataV2: RouterTestData[] = [ + // { + // id: 'klaviyo-router-150624-test-1', + // name: 'klaviyo', + // description: '150624 -> Basic Router Test to test multiple payloads', + // scenario: 'Framework', + // successCriteria: + // 'All the subscription events from same type of call should be batched. This case does not contain any events which can be batched', + // feature: 'router', + // module: 'destination', + // version: 'v0', + // input: { + // request: { + // body: routerRequestV2, + // }, + // }, + // output: { + // response: { + // status: 200, + // body: { + // output: [ + // { + // // identify 1 + // batchedRequest: { + // version: '1', + // type: 'REST', + // method: 'POST', + // endpoint: userProfileCommonEndpoint, + // headers, + // params: {}, + // body: { + // JSON: { + // data: { + // type: 'profile', + // attributes: { + // external_id: 'test', + // email: 'test_1@rudderstack.com', + // first_name: 'Test', + // last_name: 'Rudderlabs', + // phone_number: '+12 345 578 900', + // anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + // title: 'Developer', + // organization: 'Rudder', + // location: { + // city: 'Tokyo', + // region: 'Kanto', + // country: 'JP', + // zip: '100-0001', + // }, + // properties: { Flagged: false, Residence: 'Shibuya' }, + // }, + // meta: { + // patch_properties: {}, + // }, + // }, + // }, + // JSON_ARRAY: {}, + // XML: {}, + // FORM: {}, + // }, + // files: {}, + // }, + // metadata: [generateMetadata(1)], + // batched: false, + // statusCode: 200, + // destination, + // }, + // { + // // identify 2 with subscriptipon request + // batchedRequest: [ + // { + // version: '1', + // type: 'REST', + // method: 'POST', + // endpoint: userProfileCommonEndpoint, + // headers, + // params: {}, + // body: { + // JSON: { + // data: { + // type: 'profile', + // attributes: { + // external_id: 'test', + // email: 'test@rudderstack.com', + // first_name: 'Test', + // last_name: 'Rudderlabs', + // phone_number: '+12 345 578 900', + // anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + // title: 'Developer', + // organization: 'Rudder', + // location: { + // city: 'Tokyo', + // region: 'Kanto', + // country: 'JP', + // zip: '100-0001', + // }, + // properties: { Flagged: false, Residence: 'Shibuya' }, + // }, + // meta: { + // patch_properties: {}, + // }, + // }, + // }, + // JSON_ARRAY: {}, + // XML: {}, + // FORM: {}, + // }, + // files: {}, + // }, + // { + // version: '1', + // type: 'REST', + // method: 'POST', + // endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + // headers, + // params: {}, + // body: { + // JSON: { + // data: { + // type: 'profile-subscription-bulk-create-job', + // attributes: { + // profiles: { + // data: [ + // { + // type: 'profile', + // attributes: { + // email: 'test@rudderstack.com', + // phone_number: '+12 345 578 900', + // subscriptions: { + // email: { marketing: { consent: 'SUBSCRIBED' } }, + // sms: { marketing: { consent: 'SUBSCRIBED' } }, + // }, + // }, + // }, + // ], + // }, + // }, + // relationships: subscriptionRelations, + // }, + // }, + // JSON_ARRAY: {}, + // XML: {}, + // FORM: {}, + // }, + // files: {}, + // }, + // ], + // metadata: [generateMetadata(2)], + // batched: true, + // statusCode: 200, + // destination, + // }, + // { + // // group call subscription request + // batchedRequest: [ + // { + // version: '1', + // type: 'REST', + // method: 'POST', + // endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + // headers, + // params: {}, + // body: { + // JSON: { + // data: { + // type: 'profile-subscription-bulk-create-job', + // attributes: { + // profiles: { + // data: [ + // { + // type: 'profile', + // attributes: { + // email: 'test@rudderstack.com', + // phone_number: '+12 345 678 900', + // subscriptions: { + // email: { marketing: { consent: 'SUBSCRIBED' } }, + // }, + // }, + // }, + // ], + // }, + // }, + // relationships: subscriptionRelations, + // }, + // }, + // JSON_ARRAY: {}, + // XML: {}, + // FORM: {}, + // }, + // files: {}, + // }, + // ], + // metadata: [generateMetadata(3)], + // batched: true, + // statusCode: 200, + // destination, + // }, + // { + // metadata: [generateMetadata(4)], + // batched: false, + // statusCode: 400, + // error: 'Event type random is not supported', + // statTags: { + // destType: 'KLAVIYO', + // errorCategory: 'dataValidation', + // errorType: 'instrumentation', + // feature: 'router', + // implementation: 'native', + // module: 'destination', + // destinationId: 'default-destinationId', + // workspaceId: 'default-workspaceId', + // }, + // destination, + // }, + // { + // metadata: [generateMetadata(5)], + // batched: false, + // statusCode: 400, + // error: 'groupId is a required field for group events', + // statTags: { + // destType: 'KLAVIYO', + // errorCategory: 'dataValidation', + // errorType: 'instrumentation', + // feature: 'router', + // implementation: 'native', + // module: 'destination', + // destinationId: 'default-destinationId', + // workspaceId: 'default-workspaceId', + // }, + // destination, + // }, + // ], + // }, + // }, + // }, + // }, + // { + // id: 'klaviyo-router-150624-test-2', + // name: 'klaviyo', + // description: '150624 -> Router Test to test batching based upon same message type', + // scenario: 'Framework', + // successCriteria: + // 'All the subscription events from same type of call should be batched. This case does not contain any events which can be batched', + // feature: 'router', + // module: 'destination', + // version: 'v0', + // input: { + // request: { + // body: { + // input: [ + // { + // destination, + // metadata: generateMetadata(1), + // message: { + // type: 'identify', + // sentAt: '2021-01-03T17:02:53.195Z', + // userId: 'test', + // channel: 'web', + // context: { + // os: { name: '', version: '' }, + // app: { + // name: 'RudderLabs JavaScript SDK', + // build: '1.0.0', + // version: '1.1.11', + // namespace: 'com.rudderlabs.javascript', + // }, + // traits: { + // firstName: 'Test', + // lastName: 'Rudderlabs', + // email: 'test_1@rudderstack.com', + // phone: '+12 345 578 900', + // userId: 'Testc', + // title: 'Developer', + // organization: 'Rudder', + // city: 'Tokyo', + // region: 'Kanto', + // country: 'JP', + // zip: '100-0001', + // Flagged: false, + // Residence: 'Shibuya', + // properties: { listId: 'XUepkK', subscribe: true, consent: ['email'] }, + // }, + // locale: 'en-US', + // screen: { density: 2 }, + // library: { name: 'RudderLabs JavaScript SDK', version: '1.1.11' }, + // campaign: {}, + // userAgent: + // 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', + // }, + // rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + // messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + // anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + // integrations: { All: true }, + // originalTimestamp: '2021-01-03T17:02:53.193Z', + // }, + // }, + // { + // destination, + // metadata: generateMetadata(2), + // message: { + // type: 'identify', + // sentAt: '2021-01-03T17:02:53.195Z', + // userId: 'test', + // channel: 'web', + // context: { + // os: { name: '', version: '' }, + // app: { + // name: 'RudderLabs JavaScript SDK', + // build: '1.0.0', + // version: '1.1.11', + // namespace: 'com.rudderlabs.javascript', + // }, + // traits: { + // firstName: 'Test', + // lastName: 'Rudderlabs', + // email: 'test@rudderstack.com', + // phone: '+12 345 578 900', + // userId: 'test', + // title: 'Developer', + // organization: 'Rudder', + // city: 'Tokyo', + // region: 'Kanto', + // country: 'JP', + // zip: '100-0001', + // Flagged: false, + // Residence: 'Shibuya', + // properties: { listId: 'XUepkK', subscribe: true, consent: ['email', 'sms'] }, + // }, + // locale: 'en-US', + // screen: { density: 2 }, + // library: { name: 'RudderLabs JavaScript SDK', version: '1.1.11' }, + // campaign: {}, + // userAgent: + // 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', + // }, + // rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + // messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + // anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + // integrations: { All: true }, + // originalTimestamp: '2021-01-03T17:02:53.193Z', + // }, + // }, + // ], + // destType: 'klaviyo', + // }, + // }, + // }, + // output: { + // response: { + // status: 200, + // body: { + // output: [ + // { + // // profile for identify 1 + // batchedRequest: [ + // { + // version: '1', + // type: 'REST', + // method: 'POST', + // endpoint: userProfileCommonEndpoint, + // headers, + // params: {}, + // body: { + // JSON: { + // data: { + // type: 'profile', + // attributes: { + // external_id: 'test', + // email: 'test_1@rudderstack.com', + // first_name: 'Test', + // last_name: 'Rudderlabs', + // phone_number: '+12 345 578 900', + // anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + // title: 'Developer', + // organization: 'Rudder', + // location: { + // city: 'Tokyo', + // region: 'Kanto', + // country: 'JP', + // zip: '100-0001', + // }, + // properties: { Flagged: false, Residence: 'Shibuya' }, + // }, + // meta: { + // patch_properties: {}, + // }, + // }, + // }, + // JSON_ARRAY: {}, + // XML: {}, + // FORM: {}, + // }, + // files: {}, + // }, + // // profile for identify 2 + // { + // version: '1', + // type: 'REST', + // method: 'POST', + // endpoint: userProfileCommonEndpoint, + // headers, + // params: {}, + // body: { + // JSON: { + // data: { + // type: 'profile', + // attributes: { + // external_id: 'test', + // email: 'test@rudderstack.com', + // first_name: 'Test', + // last_name: 'Rudderlabs', + // phone_number: '+12 345 578 900', + // anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + // title: 'Developer', + // organization: 'Rudder', + // location: { + // city: 'Tokyo', + // region: 'Kanto', + // country: 'JP', + // zip: '100-0001', + // }, + // properties: { Flagged: false, Residence: 'Shibuya' }, + // }, + // meta: { + // patch_properties: {}, + // }, + // }, + // }, + // JSON_ARRAY: {}, + // XML: {}, + // FORM: {}, + // }, + // files: {}, + // }, + // // subscriptiopn for both identify 1 and 2 + // { + // version: '1', + // type: 'REST', + // method: 'POST', + // endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + // headers, + // params: {}, + // body: { + // JSON: { + // data: { + // type: 'profile-subscription-bulk-create-job', + // attributes: { + // profiles: { + // data: [ + // { + // type: 'profile', + // attributes: { + // email: 'test_1@rudderstack.com', + // phone_number: '+12 345 578 900', + // subscriptions: { + // email: { marketing: { consent: 'SUBSCRIBED' } }, + // }, + // }, + // }, + // { + // type: 'profile', + // attributes: { + // email: 'test@rudderstack.com', + // phone_number: '+12 345 578 900', + // subscriptions: { + // email: { marketing: { consent: 'SUBSCRIBED' } }, + // sms: { marketing: { consent: 'SUBSCRIBED' } }, + // }, + // }, + // }, + // ], + // }, + // }, + // relationships: subscriptionRelations, + // }, + // }, + // JSON_ARRAY: {}, + // XML: {}, + // FORM: {}, + // }, + // files: {}, + // }, + // ], + // metadata: [generateMetadata(1), generateMetadata(2)], + // batched: true, + // statusCode: 200, + // destination, + // }, + // ], + // }, + // }, + // }, + // }, { - id: 'klaviyo-router-150624-test-1', - name: 'klaviyo', - description: '150624 -> Basic Router Test to test multiple payloads', - scenario: 'Framework', - successCriteria: 'All the subscription events should be batched', - feature: 'router', - module: 'destination', - version: 'v0', - input: { - request: { - body: routerRequestV2, - }, - }, - output: { - response: { - status: 200, - body: { - output: [ - { - batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: userProfileCommonEndpoint, - headers, - params: {}, - body: { - JSON: { - data: { - type: 'profile', - attributes: { - external_id: 'test', - email: 'test_1@rudderstack.com', - first_name: 'Test', - last_name: 'Rudderlabs', - phone_number: '+12 345 578 900', - anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', - title: 'Developer', - organization: 'Rudder', - location: { - city: 'Tokyo', - region: 'Kanto', - country: 'JP', - zip: '100-0001', - }, - properties: { Flagged: false, Residence: 'Shibuya' }, - }, - meta: { - patch_properties: {}, - }, - }, - }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, - }, - files: {}, - }, - metadata: [generateMetadata(1)], - batched: false, - statusCode: 200, - destination, - }, - { - batchedRequest: [ - { - version: '1', - type: 'REST', - method: 'POST', - endpoint: userProfileCommonEndpoint, - headers, - params: {}, - body: { - JSON: { - data: { - type: 'profile', - attributes: { - external_id: 'test', - email: 'test@rudderstack.com', - first_name: 'Test', - last_name: 'Rudderlabs', - phone_number: '+12 345 578 900', - anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', - title: 'Developer', - organization: 'Rudder', - location: { - city: 'Tokyo', - region: 'Kanto', - country: 'JP', - zip: '100-0001', - }, - properties: { Flagged: false, Residence: 'Shibuya' }, - }, - meta: { - patch_properties: {}, - }, - }, - }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, - }, - files: {}, - }, - { - version: '1', - type: 'REST', - method: 'POST', - endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', - headers, - params: {}, - body: { - JSON: { - data: { - type: 'profile-subscription-bulk-create-job', - attributes: { - profiles: { - data: [ - { - type: 'profile', - attributes: { - email: 'test@rudderstack.com', - phone_number: '+12 345 578 900', - subscriptions: { - email: { marketing: { consent: 'SUBSCRIBED' } }, - sms: { marketing: { consent: 'SUBSCRIBED' } }, - }, - }, - }, - { - type: 'profile', - attributes: { - email: 'test@rudderstack.com', - phone_number: '+12 345 678 900', - subscriptions: { - email: { marketing: { consent: 'SUBSCRIBED' } }, - }, - }, - }, - ], - }, - }, - relationships: subscriptionRelations, - }, - }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, - }, - files: {}, - }, - ], - metadata: [generateMetadata(2), generateMetadata(3)], - batched: true, - statusCode: 200, - destination, - }, - { - metadata: [generateMetadata(4)], - batched: false, - statusCode: 400, - error: 'Event type random is not supported', - statTags: { - destType: 'KLAVIYO', - errorCategory: 'dataValidation', - errorType: 'instrumentation', - feature: 'router', - implementation: 'native', - module: 'destination', - destinationId: 'default-destinationId', - workspaceId: 'default-workspaceId', - }, - destination, - }, - { - metadata: [generateMetadata(5)], - batched: false, - statusCode: 400, - error: 'groupId is a required field for group events', - statTags: { - destType: 'KLAVIYO', - errorCategory: 'dataValidation', - errorType: 'instrumentation', - feature: 'router', - implementation: 'native', - module: 'destination', - destinationId: 'default-destinationId', - workspaceId: 'default-workspaceId', - }, - destination, - }, - ], - }, - }, - }, - }, - { - id: 'klaviyo-router-150624-test-2', + id: 'klaviyo-router-150624-test-3', name: 'klaviyo', description: '150624 -> Router tests to have some anonymous track event, some identify events with subscription and some identified track event', scenario: 'Framework', successCriteria: - 'All the subscription events should be batched and respective profile requests should also be placed in same batched request', + 'All the subscription events under same message type should be batched and respective profile requests should also be placed in same batched request', feature: 'router', module: 'destination', version: 'v0', @@ -307,7 +602,7 @@ export const dataV2: RouterTestData[] = [ price: '12', }, }, - metadata: generateMetadata(1), + metadata: generateMetadata(1, 'testKlaviyo1'), destination, }, { @@ -348,7 +643,7 @@ export const dataV2: RouterTestData[] = [ All: true, }, }, - metadata: generateMetadata(3), + metadata: generateMetadata(3, 'testKlaviyo2'), destination, }, { @@ -368,7 +663,7 @@ export const dataV2: RouterTestData[] = [ price: '120', }, }, - metadata: generateMetadata(4), + metadata: generateMetadata(4, 'testKlaviyo2'), destination, }, { @@ -391,28 +686,7 @@ export const dataV2: RouterTestData[] = [ All: true, }, }, - metadata: generateMetadata(5), - destination, - }, - { - message: { - channel: 'web', - context: { - traits: { - email: 'testklaviyo3@email.com', - firstname: 'Test klaviyo3', - anonymousId: '1111', - }, - }, - type: 'identify', - anonymousId: '', - userId: 'testKlaviyo3', - integrations: { - All: true, - }, - sentAt: '2019-10-14T09:03:22.563Z', - }, - metadata: generateMetadata(6), + metadata: generateMetadata(5, 'testKlaviyo3'), destination, }, ], @@ -445,6 +719,7 @@ export const dataV2: RouterTestData[] = [ destination, }, { + // user 1 track call with userId and anonymousId batchedRequest: { version: '1', type: 'REST', @@ -463,13 +738,16 @@ export const dataV2: RouterTestData[] = [ type: 'event', attributes: { properties: { - price: '120', + price: '12', }, profile: { data: { type: 'profile', attributes: { - anonymous_id: 'anonTestKlaviyo2', + external_id: 'testKlaviyo1', + anonymous_id: 'anonTestKlaviyo1', + email: 'testklaviyo1@email.com', + first_name: 'Test Klaviyo 1', properties: {}, meta: { patch_properties: {}, @@ -481,7 +759,7 @@ export const dataV2: RouterTestData[] = [ data: { type: 'metric', attributes: { - name: 'viewed product', + name: 'purchase', }, }, }, @@ -494,17 +772,18 @@ export const dataV2: RouterTestData[] = [ }, files: {}, }, - metadata: [generateMetadata(2)], + metadata: [generateMetadata(1, 'testKlaviyo1')], batched: false, statusCode: 200, destination, }, { + // anonn event for user 2 batchedRequest: { version: '1', type: 'REST', method: 'POST', - endpoint: 'https://a.klaviyo.com/api/profile-import', + endpoint: 'https://a.klaviyo.com/api/events', headers: { Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', Accept: 'application/json', @@ -515,15 +794,31 @@ export const dataV2: RouterTestData[] = [ body: { JSON: { data: { - type: 'profile', + type: 'event', attributes: { - external_id: 'testKlaviyo3', - email: 'testklaviyo3@email.com', - first_name: 'Test klaviyo3', - properties: {}, - }, - meta: { - patch_properties: {}, + properties: { + price: '120', + }, + profile: { + data: { + type: 'profile', + attributes: { + anonymous_id: 'anonTestKlaviyo2', + properties: {}, + meta: { + patch_properties: {}, + }, + }, + }, + }, + metric: { + data: { + type: 'metric', + attributes: { + name: 'viewed product', + }, + }, + }, }, }, }, @@ -533,12 +828,13 @@ export const dataV2: RouterTestData[] = [ }, files: {}, }, - metadata: [generateMetadata(6)], + metadata: [generateMetadata(2)], batched: false, statusCode: 200, destination, }, { + // identify call for user 2 and user 3 with subscription batchedRequest: [ { version: '1', @@ -672,69 +968,11 @@ export const dataV2: RouterTestData[] = [ files: {}, }, ], - metadata: [generateMetadata(3), generateMetadata(5)], + metadata: [generateMetadata(3, 'testKlaviyo2'), generateMetadata(5, 'testKlaviyo3')], batched: true, statusCode: 200, destination, }, - { - batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: 'https://a.klaviyo.com/api/events', - headers: { - Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', - Accept: 'application/json', - 'Content-Type': 'application/json', - revision: '2024-06-15', - }, - params: {}, - body: { - JSON: { - data: { - type: 'event', - attributes: { - properties: { - price: '12', - }, - profile: { - data: { - type: 'profile', - attributes: { - external_id: 'testKlaviyo1', - anonymous_id: 'anonTestKlaviyo1', - email: 'testklaviyo1@email.com', - first_name: 'Test Klaviyo 1', - properties: {}, - meta: { - patch_properties: {}, - }, - }, - }, - }, - metric: { - data: { - type: 'metric', - attributes: { - name: 'purchase', - }, - }, - }, - }, - }, - }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, - }, - files: {}, - }, - metadata: [generateMetadata(1)], - batched: false, - statusCode: 200, - destination, - }, { batchedRequest: { version: '1', @@ -787,7 +1025,7 @@ export const dataV2: RouterTestData[] = [ }, files: {}, }, - metadata: [generateMetadata(4)], + metadata: [generateMetadata(4, 'testKlaviyo2')], batched: false, statusCode: 200, destination, @@ -803,7 +1041,7 @@ export const dataV2: RouterTestData[] = [ description: '150624 -> Retl Router tests to have retl ', scenario: 'Framework', successCriteria: - 'All the subscription events should be batched and respective profile requests should also be placed in same batched request', + 'All the subscription events with same message type should be batched and respective profile requests should also be placed in same batched request', feature: 'router', module: 'destination', version: 'v0', diff --git a/test/integrations/testUtils.ts b/test/integrations/testUtils.ts index 5e0df874f8..e5e04a2340 100644 --- a/test/integrations/testUtils.ts +++ b/test/integrations/testUtils.ts @@ -563,11 +563,11 @@ export const validateTestWithZOD = (testPayload: TestCaseData, response: any) => // ----------------------------- // Helper functions -export const generateMetadata = (jobId: number): any => { +export const generateMetadata = (jobId: number, userId?: string): any => { return { jobId, attemptNum: 1, - userId: 'default-userId', + userId: userId || 'default-userId', sourceId: 'default-sourceId', destinationId: 'default-destinationId', workspaceId: 'default-workspaceId', From c9a48377656ca60e5eab881337a8e3911434b121 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 25 Jul 2024 14:15:00 +0530 Subject: [PATCH 22/26] fix: batch logic to be more clear --- src/v0/destinations/klaviyo/batchUtil.js | 284 ++--- src/v0/destinations/klaviyo/batchUtil.test.js | 186 +++ .../destinations/klaviyo/klaviyoUtil.test.js | 71 ++ src/v0/destinations/klaviyo/transformV2.js | 11 +- src/v0/destinations/klaviyo/util.test.js | 105 -- .../destinations/klaviyo/router/dataV2.ts | 1070 +++++++++-------- 6 files changed, 947 insertions(+), 780 deletions(-) create mode 100644 src/v0/destinations/klaviyo/batchUtil.test.js delete mode 100644 src/v0/destinations/klaviyo/util.test.js diff --git a/src/v0/destinations/klaviyo/batchUtil.js b/src/v0/destinations/klaviyo/batchUtil.js index 90b36de513..0aedfcfd30 100644 --- a/src/v0/destinations/klaviyo/batchUtil.js +++ b/src/v0/destinations/klaviyo/batchUtil.js @@ -1,8 +1,13 @@ const lodash = require('lodash'); -const { defaultBatchRequestConfig, getSuccessRespEvents } = require('../../util'); +const { + defaultBatchRequestConfig, + getSuccessRespEvents, + isDefinedAndNotNull, +} = require('../../util'); const { JSON_MIME_TYPE } = require('../../util/constant'); const { BASE_ENDPOINT, CONFIG_CATEGORIES, MAX_BATCH_SIZE, revision } = require('./config'); const { buildRequest, getSubscriptionPayload } = require('./util'); + /** * This function groups the subscription responses on list id * @param {*} subscribeResponseList @@ -22,165 +27,176 @@ const groupSubscribeResponsesUsingListIdV2 = (subscribeResponseList) => { }; /** - * This function returns the list of profileReq which do not metadata common with subcriptionMetadataArray - * @param {*} profileReq - * @param {*} subscriptionMetadataArray - * @returns - */ -const getRemainingProfiles = (profileReq, subscriptionMetadataArray) => { - const subscriptionListJobIds = subscriptionMetadataArray.map((metadata) => metadata.jobId); - return profileReq.filter((profile) => !subscriptionListJobIds.includes(profile.metadata.jobId)); -}; - -/** - * This function builds all the profile requests whose metadata is not there in subscriptionMetadataArray - * @param {*} profileRespList - * @param {*} subscriptionMetadataArray - * @param {*} destination - * @returns - */ -const getProfiles = (profileRespList, subscriptionMetadataArray, destination) => { - const profiles = []; - const remainingProfileReq = getRemainingProfiles(profileRespList, subscriptionMetadataArray); - remainingProfileReq.forEach((input) => { - profiles.push( - getSuccessRespEvents( - buildRequest(input.payload, destination, CONFIG_CATEGORIES.IDENTIFYV2), - [input.metadata], - destination, - ), - ); - }); - return profiles; -}; - -/** - * This function takes susbscriptions as input and batches them into a single request body - * @param {events} - * events= [ - * { payload: {id:'list_id', profile: {}}, metadata:{} }, - * { payload: {id:'list_id', profile: {}}, metadata:{} } - * ] + * This function takes susbscription as input and batches them into a single request body + * @param {subscription} + * subscription= {listId, subscriptionProfileList} */ -const generateBatchedSubscriptionRequest = (events, destination) => { +const generateBatchedSubscriptionRequest = (subscription, destination) => { const batchEventResponse = defaultBatchRequestConfig(); - const metadata = []; + // if( !isDefinedAndNotNull(subscription) ) // fetching listId from first event as listId is same for all the events - const listId = events[0].payload?.listId; - const profiles = []; // list of profiles to be subscribes - // Batch profiles into dest batch structure - events.forEach((ev) => { - profiles.push(...ev.payload.profile); - metadata.push(ev.metadata); - }); - - batchEventResponse.batchedRequest = Object.values(batchEventResponse); - batchEventResponse.batchedRequest[0].body.JSON = getSubscriptionPayload(listId, profiles); - - batchEventResponse.batchedRequest[0].endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; - - batchEventResponse.batchedRequest[0].headers = { + const profiles = []; // list of profiles to be subscribed + const { listId, subscriptionProfileList } = subscription; + subscriptionProfileList.forEach((profileList) => profiles.push(...profileList)); + batchEventResponse.batchedRequest.body.JSON = getSubscriptionPayload(listId, profiles); + batchEventResponse.batchedRequest.endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; + batchEventResponse.batchedRequest.headers = { Authorization: `Klaviyo-API-Key ${destination.Config.privateApiKey}`, 'Content-Type': JSON_MIME_TYPE, Accept: JSON_MIME_TYPE, revision, }; - - return { - ...batchEventResponse, - metadata, - destination, - }; + return batchEventResponse.batchedRequest; }; /** - * This function fetches the profileRequests with metadata present in metadata array build a request for them - * and add these requests batchEvent Response - * @param {*} profileReq array of profile requests - * @param {*} metadataArray array of metadata - * @param {*} batchEventResponse - * Example: /** - * - * @param {*} subscribeEventGroups - * @param {*} identifyResponseList - * @returns - * Example: - * profileReq = [ - * { payload: {}, metadata:{} }, - * { payload: {}, metadata:{} } - * ] + * This function updates batchedRequest with profile requests + * @param {*} profiles + * @param {*} batchedRequest + * @param {*} destination */ -const updateBatchEventResponseWithProfileRequests = ( - profileReqArr, - subscriptionMetadataArray, - batchEventResponse, -) => { - const subscriptionListJobIds = subscriptionMetadataArray.map((metadata) => metadata.jobId); +const updateBatchEventResponseWithProfileRequests = (profiles, batchedRequest, destination) => { const profilesRequests = []; - profileReqArr.forEach((profile) => { - if (subscriptionListJobIds.includes(profile.metadata.jobId)) { - profilesRequests.push( - buildRequest(profile.payload, batchEventResponse.destination, CONFIG_CATEGORIES.IDENTIFYV2), - ); - } + profiles.forEach((profile) => { + profilesRequests.push(buildRequest(profile, destination, CONFIG_CATEGORIES.IDENTIFYV2)); }); - // we are keeping profiles request prior to subscription ones - batchEventResponse.batchedRequest.unshift(...profilesRequests); + // we are keeping profiles request prior to subscription ones as first profile creation and then subscription should happen + batchedRequest.unshift(...profilesRequests); }; -const processSubscribeChunk = (chunk, destination, profileRespList) => { - const batchEventResponse = generateBatchedSubscriptionRequest(chunk, destination); - const { metadata: subscriptionMetadataArray } = batchEventResponse; - updateBatchEventResponseWithProfileRequests( - profileRespList, - subscriptionMetadataArray, - batchEventResponse, - ); - return batchEventResponse; +/** + * this function populates profileSubscriptionAndMetadataArr with respective profiles based upon common metadata + * @param {*} profileSubscriptionAndMetadataArr + * @param {*} metadataIndexMap + * @param {*} profiles + * @returns updated profileSubscriptionAndMetadataArr obj + */ +const populateArrWithRespectiveProfileData = ( + profileSubscriptionAndMetadataArr, + metadataIndexMap, + profiles, +) => { + const updatedPSMArr = profileSubscriptionAndMetadataArr; + profiles.forEach((profile) => { + const index = metadataIndexMap[profile.metadata.jobId]; + if (isDefinedAndNotNull(index)) { + // using isDefinedAndNotNull as index can be 0 + updatedPSMArr[index].profiles.push(profile.payload); + } else { + // in case there is no subscription for a given profile + updatedPSMArr.push({ + profiles: [profile.payload], + metadataList: [profile.metadata], + }); + } + }); + return updatedPSMArr; }; /** - * This function batches the requests. Alogorithm - * Batch events from Subscribe Resp List having same listId/groupId to be subscribed and have their metadata array - * For this metadata array get all profileRequests and add them prior to batched Subscribe Request in the same batched Request - * Make another batched request for the remaning profile requests and another for all the event requests - * @param {*} subscribeRespList - * @param {*} profileRespList - * @param {*} eventRespList - * subscribeRespList = [ - * { payload: {id:'list_id', profile: {}}, metadata:{} }, - * { payload: {id:'list_id', profile: {}}, metadata:{} } - * ] - * profileRespList = [ - * { payload: {}, metadata:{} }, - * { payload: {}, metadata:{} } - * ] - * + * This function generates the final output batched payload for each object in profileSubscriptionAndMetadataArr + * ex: + * profileSubscriptionAndMetadataArr = [ + { + subscription: { subscriptionProfileList, listId1 }, + metadataList1, + profiles: [respectiveProfiles for above metadata] + }, + { + subscription: { subscriptionProfile List With No Profiles, listId2 }, + metadataList2, + }, + { + metadataList3, + profiles: [respectiveProfiles for above metadata with no subscription] + } + ] + * @param {*} profileSubscriptionAndMetadataArr + * @param {*} destination + * @returns */ -const batchSubscriptionRequestV2 = (subscribeRespList, profileRespList, destination) => { +const buildRequestsForProfileSubscriptionAndMetadataArr = ( + profileSubscriptionAndMetadataArr, + destination, +) => { const batchedResponseList = []; - const subscriptionMetadataArrayForAll = []; + profileSubscriptionAndMetadataArr.forEach((input) => { + const batchedRequest = []; + if (input.subscription) { + batchedRequest.push(generateBatchedSubscriptionRequest(input.subscription, destination)); + } + updateBatchEventResponseWithProfileRequests(input.profiles, batchedRequest, destination); + batchedResponseList.push( + getSuccessRespEvents(batchedRequest, input.metadataList, destination, true), + ); + }); + return batchedResponseList; +}; + +const batchRequestV2 = (subscribeRespList, profileRespList, destination) => { const subscribeEventGroups = groupSubscribeResponsesUsingListIdV2(subscribeRespList); + let profileSubscriptionAndMetadataArr = []; + const metadataIndexMap = {}; Object.keys(subscribeEventGroups).forEach((listId) => { // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] const eventChunks = lodash.chunk(subscribeEventGroups[listId], MAX_BATCH_SIZE); - const batchedResponse = []; - eventChunks.forEach((chunk) => { - // returns subscriptionMetadata and batchEventResponse - const { metadata: subscriptionMetadataArray, batchedRequest } = processSubscribeChunk( - chunk, - destination, - profileRespList, - ); - subscriptionMetadataArrayForAll.push(...subscriptionMetadataArray); - batchedResponse.push( - getSuccessRespEvents(batchedRequest, subscriptionMetadataArray, destination, true), - ); + eventChunks.forEach((chunk, index) => { + // get subscriptionProfiles for the chunk + const subscriptionProfileList = chunk.map((event) => event.payload?.profile); + // get metadata for this chunk + const metadataList = chunk.map((event) => event.metadata); + // get list of jobIds from the above metdadata + const jobIdList = metadataList.map((metadata) => metadata.jobId); + // push the jobId: index to metadataIndex mapping which let us know the metadata respective payload index position in batched request + jobIdList.forEach((jobId) => { + metadataIndexMap[jobId] = index; + }); + profileSubscriptionAndMetadataArr.push({ + subscription: { subscriptionProfileList, listId }, + metadataList, + profiles: [], + }); }); - batchedResponseList.push(...batchedResponse); }); - const profiles = getProfiles(profileRespList, subscriptionMetadataArrayForAll, destination); + profileSubscriptionAndMetadataArr = populateArrWithRespectiveProfileData( + profileSubscriptionAndMetadataArr, + metadataIndexMap, + profileRespList, + ); + /* Till this point I have a profileSubscriptionAndMetadataArr + containing the the events in one object for which batching has to happen in following format + [ + { + subscription: { subscriptionProfileList, listId1 }, + metadataList1, + profiles: [respectiveProfiles for above metadata] + }, + { + subscription: { subscriptionProfile List With No Profiles, listId2 }, + metadataList2, + }, + { + metadataList3, + profiles: [respectiveProfiles for above metadata with no subscription] + } + ] + */ + return buildRequestsForProfileSubscriptionAndMetadataArr( + profileSubscriptionAndMetadataArr, + destination, + ); + /* for identify calls with batching batched with identify with no batching + we will sonctruct O/P as: + [ + [2 calls for identifywith batching], + [1 call identify calls with batching] + ] + */ +}; - return [...profiles, ...batchedResponseList]; +module.exports = { + groupSubscribeResponsesUsingListIdV2, + populateArrWithRespectiveProfileData, + generateBatchedSubscriptionRequest, + batchRequestV2, }; -module.exports = { batchSubscriptionRequestV2, groupSubscribeResponsesUsingListIdV2 }; diff --git a/src/v0/destinations/klaviyo/batchUtil.test.js b/src/v0/destinations/klaviyo/batchUtil.test.js new file mode 100644 index 0000000000..ccaaf840cd --- /dev/null +++ b/src/v0/destinations/klaviyo/batchUtil.test.js @@ -0,0 +1,186 @@ +const { + groupSubscribeResponsesUsingListIdV2, + populateArrWithRespectiveProfileData, + generateBatchedSubscriptionRequest, +} = require('./batchUtil'); +const { revision } = require('./config'); + +describe('groupSubscribeResponsesUsingListIdV2', () => { + // Groups subscription responses by listId correctly + it('should group subscription responses by listId correctly when given a valid list', () => { + const subscribeResponseList = [ + { payload: { listId: 'list_1', profile: {} }, metadata: {} }, + { payload: { listId: 'list_1', profile: {} }, metadata: {} }, + { payload: { listId: 'list_2', profile: {} }, metadata: {} }, + ]; + + const expectedOutput = { + list_1: [ + { payload: { listId: 'list_1', profile: {} }, metadata: {} }, + { payload: { listId: 'list_1', profile: {} }, metadata: {} }, + ], + list_2: [{ payload: { listId: 'list_2', profile: {} }, metadata: {} }], + }; + + const result = groupSubscribeResponsesUsingListIdV2(subscribeResponseList); + + expect(result).toEqual(expectedOutput); + }); + + // Handles empty subscription response list + it('should return an empty object when given an empty subscription response list', () => { + const subscribeResponseList = []; + + const expectedOutput = {}; + + const result = groupSubscribeResponsesUsingListIdV2(subscribeResponseList); + + expect(result).toEqual(expectedOutput); + }); +}); + +describe('populateArrWithRespectiveProfileData', () => { + // Correctly populates array when all profiles have corresponding metadata + it('should correctly populate array when all profiles have corresponding metadata', () => { + const profileSubscriptionAndMetadataArr = [ + { profiles: [], metadataList: [{ jobId: '1' }], subscriptions: [] }, + { profiles: [], metadataList: [{ jobId: '2' }], subscriptions: [] }, + ]; + const metadataIndexMap = { 1: 0, 2: 1 }; + const profiles = [ + { payload: { name: 'John' }, metadata: { jobId: '1' } }, + { payload: { name: 'Doe' }, metadata: { jobId: '2' } }, + ]; + + const result = populateArrWithRespectiveProfileData( + profileSubscriptionAndMetadataArr, + metadataIndexMap, + profiles, + ); + + expect(result[0].profiles).toEqual([{ name: 'John' }]); + expect(result[1].profiles).toEqual([{ name: 'Doe' }]); + }); + + // Handles empty profileSubscriptionAndMetadataArr input + it('should handle empty profileSubscriptionAndMetadataArr input', () => { + const profileSubscriptionAndMetadataArr = []; + const metadataIndexMap = {}; + const profiles = [{ payload: { name: 'John' }, metadata: { jobId: '1' } }]; + + const result = populateArrWithRespectiveProfileData( + profileSubscriptionAndMetadataArr, + metadataIndexMap, + profiles, + ); + + expect(result).toEqual([ + { + profiles: [{ name: 'John' }], + metadataList: [{ jobId: '1' }], + }, + ]); + }); +}); + +// Generated by CodiumAI + +describe('generateBatchedSubscriptionRequest', () => { + // Generates a batched subscription request with valid subscription and destination inputs + it('should generate a valid batched subscription request when given valid subscription and destination inputs', () => { + const subscription = { + listId: 'test-list-id', + subscriptionProfileList: [[{ id: 'profile1' }, { id: 'profile2' }], [{ id: 'profile3' }]], + }; + const destination = { + Config: { + privateApiKey: 'test-api-key', + }, + }; + const expectedRequest = { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers: { + Authorization: 'Klaviyo-API-Key test-api-key', + 'Content-Type': 'application/json', + Accept: 'application/json', + revision, + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { + profiles: { data: [{ id: 'profile1' }, { id: 'profile2' }, { id: 'profile3' }] }, + }, + relationships: { + list: { + data: { + type: 'list', + id: 'test-list-id', + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }; + const result = generateBatchedSubscriptionRequest(subscription, destination); + expect(result).toEqual(expectedRequest); + }); + + // Handles empty subscriptionProfileList gracefully + it('should handle empty subscriptionProfileList gracefully', () => { + const subscription = { + listId: 'test-list-id', + subscriptionProfileList: [], + }; + const destination = { + Config: { + privateApiKey: 'test-api-key', + }, + }; + const expectedRequest = { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers: { + Authorization: 'Klaviyo-API-Key test-api-key', + 'Content-Type': 'application/json', + Accept: 'application/json', + revision, + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { profiles: { data: [] } }, + relationships: { + list: { + data: { + type: 'list', + id: 'test-list-id', + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }; + const result = generateBatchedSubscriptionRequest(subscription, destination); + expect(result).toEqual(expectedRequest); + }); +}); diff --git a/src/v0/destinations/klaviyo/klaviyoUtil.test.js b/src/v0/destinations/klaviyo/klaviyoUtil.test.js index f64ea8335a..051a6c3599 100644 --- a/src/v0/destinations/klaviyo/klaviyoUtil.test.js +++ b/src/v0/destinations/klaviyo/klaviyoUtil.test.js @@ -154,3 +154,74 @@ describe('addSubscribeFlagToTraits', () => { expect(result.properties.subscribe).toBe(true); }); }); + +const { getProfileMetadataAndMetadataFields } = require('./util'); + +describe('getProfileMetadataAndMetadataFields', () => { + // Correctly generates metadata with fields to unset, append, and unappend when all fields are provided + it('should generate metadata with fields to unset, append, and unappend when all fields are provided', () => { + const message = { + integrations: { + Klaviyo: { + fieldsToUnset: ['Unset1', 'Unset2'], + fieldsToAppend: ['appendList1', 'appendList2'], + fieldsToUnappend: ['unappendList1', 'unappendList2'], + }, + All: true, + }, + traits: { + appendList1: 'New Value 1', + appendList2: 'New Value 2', + unappendList1: 'Old Value 1', + unappendList2: 'Old Value 2', + }, + }; + + jest.mock('../../util', () => ({ + getIntegrationsObj: jest.fn(() => message.integrations.Klaviyo), + getFieldValueFromMessage: jest.fn(() => message.traits), + isDefinedAndNotNull: jest.fn((value) => value !== null && value !== undefined), + })); + + const result = getProfileMetadataAndMetadataFields(message); + + expect(result).toEqual({ + meta: { + patch_properties: { + append: { + appendList1: 'New Value 1', + appendList2: 'New Value 2', + }, + unappend: { + unappendList1: 'Old Value 1', + unappendList2: 'Old Value 2', + }, + unset: ['Unset1', 'Unset2'], + }, + }, + metadataFields: [ + 'Unset1', + 'Unset2', + 'appendList1', + 'appendList2', + 'unappendList1', + 'unappendList2', + ], + }); + }); + + // Handles null or undefined message input gracefully + it('should return empty metadata and metadataFields when message is null or undefined', () => { + jest.mock('../../util', () => ({ + getIntegrationsObj: jest.fn(() => null), + getFieldValueFromMessage: jest.fn(() => ({})), + isDefinedAndNotNull: jest.fn((value) => value !== null && value !== undefined), + })); + + let result = getProfileMetadataAndMetadataFields(null); + expect(result).toEqual({ meta: { patch_properties: {} }, metadataFields: [] }); + + result = getProfileMetadataAndMetadataFields(undefined); + expect(result).toEqual({ meta: { patch_properties: {} }, metadataFields: [] }); + }); +}); diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index 5982729315..ad98d2f559 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -14,7 +14,7 @@ const { fetchTransformedEvents, addSubscribeFlagToTraits, } = require('./util'); -const { batchSubscriptionRequestV2 } = require('./batchUtil'); +const { batchRequestV2 } = require('./batchUtil'); const { constructPayload, getFieldValueFromMessage, @@ -108,7 +108,7 @@ const groupRequestHandler = (message, category, destination) => { if (!traitsInfo?.subscribe) { throw new InstrumentationError('Subscribe flag should be true for group call'); } - + // throwing error for subscribe flag return { subscription: subscribeUserToListV2(message, traitsInfo, destination) }; }; @@ -204,14 +204,9 @@ const processRouter = (inputs, reqMetadata) => { batchErrorRespList.push(errRespEvent); } }); - const batchedResponseList = batchSubscriptionRequestV2( - subscribeRespList, - profileRespList, - destination, - ); + const batchedResponseList = batchRequestV2(subscribeRespList, profileRespList, destination); const trackRespList = getTrackRequests(eventRespList, destination); - // We are doing to maintain event ordering basically once a user is identified klaviyo does not allow user tracking based upon anonymous_id only batchResponseList.push(...trackRespList, ...batchedResponseList); return { successEvents: batchResponseList, errorEvents: batchErrorRespList }; diff --git a/src/v0/destinations/klaviyo/util.test.js b/src/v0/destinations/klaviyo/util.test.js deleted file mode 100644 index 755c937cff..0000000000 --- a/src/v0/destinations/klaviyo/util.test.js +++ /dev/null @@ -1,105 +0,0 @@ -const { getProfileMetadataAndMetadataFields } = require('./util'); - -const { groupSubscribeResponsesUsingListIdV2 } = require('./batchUtil'); -describe('getProfileMetadataAndMetadataFields', () => { - // Correctly generates metadata with fields to unset, append, and unappend when all fields are provided - it('should generate metadata with fields to unset, append, and unappend when all fields are provided', () => { - const message = { - integrations: { - Klaviyo: { - fieldsToUnset: ['Unset1', 'Unset2'], - fieldsToAppend: ['appendList1', 'appendList2'], - fieldsToUnappend: ['unappendList1', 'unappendList2'], - }, - All: true, - }, - traits: { - appendList1: 'New Value 1', - appendList2: 'New Value 2', - unappendList1: 'Old Value 1', - unappendList2: 'Old Value 2', - }, - }; - - jest.mock('../../util', () => ({ - getIntegrationsObj: jest.fn(() => message.integrations.Klaviyo), - getFieldValueFromMessage: jest.fn(() => message.traits), - isDefinedAndNotNull: jest.fn((value) => value !== null && value !== undefined), - })); - - const result = getProfileMetadataAndMetadataFields(message); - - expect(result).toEqual({ - meta: { - patch_properties: { - append: { - appendList1: 'New Value 1', - appendList2: 'New Value 2', - }, - unappend: { - unappendList1: 'Old Value 1', - unappendList2: 'Old Value 2', - }, - unset: ['Unset1', 'Unset2'], - }, - }, - metadataFields: [ - 'Unset1', - 'Unset2', - 'appendList1', - 'appendList2', - 'unappendList1', - 'unappendList2', - ], - }); - }); - - // Handles null or undefined message input gracefully - it('should return empty metadata and metadataFields when message is null or undefined', () => { - jest.mock('../../util', () => ({ - getIntegrationsObj: jest.fn(() => null), - getFieldValueFromMessage: jest.fn(() => ({})), - isDefinedAndNotNull: jest.fn((value) => value !== null && value !== undefined), - })); - - let result = getProfileMetadataAndMetadataFields(null); - expect(result).toEqual({ meta: { patch_properties: {} }, metadataFields: [] }); - - result = getProfileMetadataAndMetadataFields(undefined); - expect(result).toEqual({ meta: { patch_properties: {} }, metadataFields: [] }); - }); -}); - -describe('groupSubscribeResponsesUsingListIdV2', () => { - // Groups subscription responses by listId correctly - it('should group subscription responses by listId correctly when given a valid list', () => { - const subscribeResponseList = [ - { payload: { listId: 'list_1', profile: {} }, metadata: {} }, - { payload: { listId: 'list_1', profile: {} }, metadata: {} }, - { payload: { listId: 'list_2', profile: {} }, metadata: {} }, - ]; - - const expectedOutput = { - list_1: [ - { payload: { listId: 'list_1', profile: {} }, metadata: {} }, - { payload: { listId: 'list_1', profile: {} }, metadata: {} }, - ], - list_2: [{ payload: { listId: 'list_2', profile: {} }, metadata: {} }], - }; - - const result = groupSubscribeResponsesUsingListIdV2(subscribeResponseList); - - expect(result).toEqual(expectedOutput); - }); - - // Handles empty subscription response list - it('should return an empty object when given an empty subscription response list', () => { - const subscribeResponseList = []; - - const expectedOutput = {}; - - const result = groupSubscribeResponsesUsingListIdV2(subscribeResponseList); - - expect(result).toEqual(expectedOutput); - }); -}); diff --git a/test/integrations/destinations/klaviyo/router/dataV2.ts b/test/integrations/destinations/klaviyo/router/dataV2.ts index d385ec5d64..714560fdfd 100644 --- a/test/integrations/destinations/klaviyo/router/dataV2.ts +++ b/test/integrations/destinations/klaviyo/router/dataV2.ts @@ -76,498 +76,500 @@ const alreadyTransformedEvent = { }; export const dataV2: RouterTestData[] = [ - // { - // id: 'klaviyo-router-150624-test-1', - // name: 'klaviyo', - // description: '150624 -> Basic Router Test to test multiple payloads', - // scenario: 'Framework', - // successCriteria: - // 'All the subscription events from same type of call should be batched. This case does not contain any events which can be batched', - // feature: 'router', - // module: 'destination', - // version: 'v0', - // input: { - // request: { - // body: routerRequestV2, - // }, - // }, - // output: { - // response: { - // status: 200, - // body: { - // output: [ - // { - // // identify 1 - // batchedRequest: { - // version: '1', - // type: 'REST', - // method: 'POST', - // endpoint: userProfileCommonEndpoint, - // headers, - // params: {}, - // body: { - // JSON: { - // data: { - // type: 'profile', - // attributes: { - // external_id: 'test', - // email: 'test_1@rudderstack.com', - // first_name: 'Test', - // last_name: 'Rudderlabs', - // phone_number: '+12 345 578 900', - // anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', - // title: 'Developer', - // organization: 'Rudder', - // location: { - // city: 'Tokyo', - // region: 'Kanto', - // country: 'JP', - // zip: '100-0001', - // }, - // properties: { Flagged: false, Residence: 'Shibuya' }, - // }, - // meta: { - // patch_properties: {}, - // }, - // }, - // }, - // JSON_ARRAY: {}, - // XML: {}, - // FORM: {}, - // }, - // files: {}, - // }, - // metadata: [generateMetadata(1)], - // batched: false, - // statusCode: 200, - // destination, - // }, - // { - // // identify 2 with subscriptipon request - // batchedRequest: [ - // { - // version: '1', - // type: 'REST', - // method: 'POST', - // endpoint: userProfileCommonEndpoint, - // headers, - // params: {}, - // body: { - // JSON: { - // data: { - // type: 'profile', - // attributes: { - // external_id: 'test', - // email: 'test@rudderstack.com', - // first_name: 'Test', - // last_name: 'Rudderlabs', - // phone_number: '+12 345 578 900', - // anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', - // title: 'Developer', - // organization: 'Rudder', - // location: { - // city: 'Tokyo', - // region: 'Kanto', - // country: 'JP', - // zip: '100-0001', - // }, - // properties: { Flagged: false, Residence: 'Shibuya' }, - // }, - // meta: { - // patch_properties: {}, - // }, - // }, - // }, - // JSON_ARRAY: {}, - // XML: {}, - // FORM: {}, - // }, - // files: {}, - // }, - // { - // version: '1', - // type: 'REST', - // method: 'POST', - // endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', - // headers, - // params: {}, - // body: { - // JSON: { - // data: { - // type: 'profile-subscription-bulk-create-job', - // attributes: { - // profiles: { - // data: [ - // { - // type: 'profile', - // attributes: { - // email: 'test@rudderstack.com', - // phone_number: '+12 345 578 900', - // subscriptions: { - // email: { marketing: { consent: 'SUBSCRIBED' } }, - // sms: { marketing: { consent: 'SUBSCRIBED' } }, - // }, - // }, - // }, - // ], - // }, - // }, - // relationships: subscriptionRelations, - // }, - // }, - // JSON_ARRAY: {}, - // XML: {}, - // FORM: {}, - // }, - // files: {}, - // }, - // ], - // metadata: [generateMetadata(2)], - // batched: true, - // statusCode: 200, - // destination, - // }, - // { - // // group call subscription request - // batchedRequest: [ - // { - // version: '1', - // type: 'REST', - // method: 'POST', - // endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', - // headers, - // params: {}, - // body: { - // JSON: { - // data: { - // type: 'profile-subscription-bulk-create-job', - // attributes: { - // profiles: { - // data: [ - // { - // type: 'profile', - // attributes: { - // email: 'test@rudderstack.com', - // phone_number: '+12 345 678 900', - // subscriptions: { - // email: { marketing: { consent: 'SUBSCRIBED' } }, - // }, - // }, - // }, - // ], - // }, - // }, - // relationships: subscriptionRelations, - // }, - // }, - // JSON_ARRAY: {}, - // XML: {}, - // FORM: {}, - // }, - // files: {}, - // }, - // ], - // metadata: [generateMetadata(3)], - // batched: true, - // statusCode: 200, - // destination, - // }, - // { - // metadata: [generateMetadata(4)], - // batched: false, - // statusCode: 400, - // error: 'Event type random is not supported', - // statTags: { - // destType: 'KLAVIYO', - // errorCategory: 'dataValidation', - // errorType: 'instrumentation', - // feature: 'router', - // implementation: 'native', - // module: 'destination', - // destinationId: 'default-destinationId', - // workspaceId: 'default-workspaceId', - // }, - // destination, - // }, - // { - // metadata: [generateMetadata(5)], - // batched: false, - // statusCode: 400, - // error: 'groupId is a required field for group events', - // statTags: { - // destType: 'KLAVIYO', - // errorCategory: 'dataValidation', - // errorType: 'instrumentation', - // feature: 'router', - // implementation: 'native', - // module: 'destination', - // destinationId: 'default-destinationId', - // workspaceId: 'default-workspaceId', - // }, - // destination, - // }, - // ], - // }, - // }, - // }, - // }, - // { - // id: 'klaviyo-router-150624-test-2', - // name: 'klaviyo', - // description: '150624 -> Router Test to test batching based upon same message type', - // scenario: 'Framework', - // successCriteria: - // 'All the subscription events from same type of call should be batched. This case does not contain any events which can be batched', - // feature: 'router', - // module: 'destination', - // version: 'v0', - // input: { - // request: { - // body: { - // input: [ - // { - // destination, - // metadata: generateMetadata(1), - // message: { - // type: 'identify', - // sentAt: '2021-01-03T17:02:53.195Z', - // userId: 'test', - // channel: 'web', - // context: { - // os: { name: '', version: '' }, - // app: { - // name: 'RudderLabs JavaScript SDK', - // build: '1.0.0', - // version: '1.1.11', - // namespace: 'com.rudderlabs.javascript', - // }, - // traits: { - // firstName: 'Test', - // lastName: 'Rudderlabs', - // email: 'test_1@rudderstack.com', - // phone: '+12 345 578 900', - // userId: 'Testc', - // title: 'Developer', - // organization: 'Rudder', - // city: 'Tokyo', - // region: 'Kanto', - // country: 'JP', - // zip: '100-0001', - // Flagged: false, - // Residence: 'Shibuya', - // properties: { listId: 'XUepkK', subscribe: true, consent: ['email'] }, - // }, - // locale: 'en-US', - // screen: { density: 2 }, - // library: { name: 'RudderLabs JavaScript SDK', version: '1.1.11' }, - // campaign: {}, - // userAgent: - // 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', - // }, - // rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', - // messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', - // anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', - // integrations: { All: true }, - // originalTimestamp: '2021-01-03T17:02:53.193Z', - // }, - // }, - // { - // destination, - // metadata: generateMetadata(2), - // message: { - // type: 'identify', - // sentAt: '2021-01-03T17:02:53.195Z', - // userId: 'test', - // channel: 'web', - // context: { - // os: { name: '', version: '' }, - // app: { - // name: 'RudderLabs JavaScript SDK', - // build: '1.0.0', - // version: '1.1.11', - // namespace: 'com.rudderlabs.javascript', - // }, - // traits: { - // firstName: 'Test', - // lastName: 'Rudderlabs', - // email: 'test@rudderstack.com', - // phone: '+12 345 578 900', - // userId: 'test', - // title: 'Developer', - // organization: 'Rudder', - // city: 'Tokyo', - // region: 'Kanto', - // country: 'JP', - // zip: '100-0001', - // Flagged: false, - // Residence: 'Shibuya', - // properties: { listId: 'XUepkK', subscribe: true, consent: ['email', 'sms'] }, - // }, - // locale: 'en-US', - // screen: { density: 2 }, - // library: { name: 'RudderLabs JavaScript SDK', version: '1.1.11' }, - // campaign: {}, - // userAgent: - // 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', - // }, - // rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', - // messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', - // anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', - // integrations: { All: true }, - // originalTimestamp: '2021-01-03T17:02:53.193Z', - // }, - // }, - // ], - // destType: 'klaviyo', - // }, - // }, - // }, - // output: { - // response: { - // status: 200, - // body: { - // output: [ - // { - // // profile for identify 1 - // batchedRequest: [ - // { - // version: '1', - // type: 'REST', - // method: 'POST', - // endpoint: userProfileCommonEndpoint, - // headers, - // params: {}, - // body: { - // JSON: { - // data: { - // type: 'profile', - // attributes: { - // external_id: 'test', - // email: 'test_1@rudderstack.com', - // first_name: 'Test', - // last_name: 'Rudderlabs', - // phone_number: '+12 345 578 900', - // anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', - // title: 'Developer', - // organization: 'Rudder', - // location: { - // city: 'Tokyo', - // region: 'Kanto', - // country: 'JP', - // zip: '100-0001', - // }, - // properties: { Flagged: false, Residence: 'Shibuya' }, - // }, - // meta: { - // patch_properties: {}, - // }, - // }, - // }, - // JSON_ARRAY: {}, - // XML: {}, - // FORM: {}, - // }, - // files: {}, - // }, - // // profile for identify 2 - // { - // version: '1', - // type: 'REST', - // method: 'POST', - // endpoint: userProfileCommonEndpoint, - // headers, - // params: {}, - // body: { - // JSON: { - // data: { - // type: 'profile', - // attributes: { - // external_id: 'test', - // email: 'test@rudderstack.com', - // first_name: 'Test', - // last_name: 'Rudderlabs', - // phone_number: '+12 345 578 900', - // anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', - // title: 'Developer', - // organization: 'Rudder', - // location: { - // city: 'Tokyo', - // region: 'Kanto', - // country: 'JP', - // zip: '100-0001', - // }, - // properties: { Flagged: false, Residence: 'Shibuya' }, - // }, - // meta: { - // patch_properties: {}, - // }, - // }, - // }, - // JSON_ARRAY: {}, - // XML: {}, - // FORM: {}, - // }, - // files: {}, - // }, - // // subscriptiopn for both identify 1 and 2 - // { - // version: '1', - // type: 'REST', - // method: 'POST', - // endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', - // headers, - // params: {}, - // body: { - // JSON: { - // data: { - // type: 'profile-subscription-bulk-create-job', - // attributes: { - // profiles: { - // data: [ - // { - // type: 'profile', - // attributes: { - // email: 'test_1@rudderstack.com', - // phone_number: '+12 345 578 900', - // subscriptions: { - // email: { marketing: { consent: 'SUBSCRIBED' } }, - // }, - // }, - // }, - // { - // type: 'profile', - // attributes: { - // email: 'test@rudderstack.com', - // phone_number: '+12 345 578 900', - // subscriptions: { - // email: { marketing: { consent: 'SUBSCRIBED' } }, - // sms: { marketing: { consent: 'SUBSCRIBED' } }, - // }, - // }, - // }, - // ], - // }, - // }, - // relationships: subscriptionRelations, - // }, - // }, - // JSON_ARRAY: {}, - // XML: {}, - // FORM: {}, - // }, - // files: {}, - // }, - // ], - // metadata: [generateMetadata(1), generateMetadata(2)], - // batched: true, - // statusCode: 200, - // destination, - // }, - // ], - // }, - // }, - // }, - // }, + { + id: 'klaviyo-router-150624-test-1', + name: 'klaviyo', + description: '150624 -> Basic Router Test to test multiple payloads', + scenario: 'Framework', + successCriteria: + 'All the subscription events from same type of call should be batched. This case does not contain any events which can be batched', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequestV2, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + // identify 2 with subscriptipon request + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'test', + email: 'test@rudderstack.com', + first_name: 'Test', + last_name: 'Rudderlabs', + phone_number: '+12 345 578 900', + anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + title: 'Developer', + organization: 'Rudder', + location: { + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + }, + properties: { Flagged: false, Residence: 'Shibuya' }, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'test@rudderstack.com', + phone_number: '+12 345 578 900', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + sms: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + ], + }, + }, + relationships: subscriptionRelations, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(2)], + batched: true, + statusCode: 200, + destination, + }, + { + // identify 1 + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'test', + email: 'test_1@rudderstack.com', + first_name: 'Test', + last_name: 'Rudderlabs', + phone_number: '+12 345 578 900', + anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + title: 'Developer', + organization: 'Rudder', + location: { + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + }, + properties: { Flagged: false, Residence: 'Shibuya' }, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(1)], + batched: true, + statusCode: 200, + destination, + }, + { + // group call subscription request + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'test@rudderstack.com', + phone_number: '+12 345 678 900', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + ], + }, + }, + relationships: subscriptionRelations, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(3)], + batched: true, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(4)], + batched: false, + statusCode: 400, + error: 'Event type random is not supported', + statTags: { + destType: 'KLAVIYO', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'native', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + }, + destination, + }, + { + metadata: [generateMetadata(5)], + batched: false, + statusCode: 400, + error: 'groupId is a required field for group events', + statTags: { + destType: 'KLAVIYO', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'native', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + }, + destination, + }, + ], + }, + }, + }, + }, + { + id: 'klaviyo-router-150624-test-2', + name: 'klaviyo', + description: '150624 -> Router Test to test batching based upon same message type', + scenario: 'Framework', + successCriteria: + 'All the subscription events from same type of call should be batched. This case does not contain any events which can be batched', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + destination, + metadata: generateMetadata(1), + message: { + type: 'identify', + sentAt: '2021-01-03T17:02:53.195Z', + userId: 'test', + channel: 'web', + context: { + os: { name: '', version: '' }, + app: { + name: 'RudderLabs JavaScript SDK', + build: '1.0.0', + version: '1.1.11', + namespace: 'com.rudderlabs.javascript', + }, + traits: { + firstName: 'Test', + lastName: 'Rudderlabs', + email: 'test_1@rudderstack.com', + phone: '+12 345 578 900', + userId: 'Testc', + title: 'Developer', + organization: 'Rudder', + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + Flagged: false, + Residence: 'Shibuya', + properties: { listId: 'XUepkK', subscribe: true, consent: ['email'] }, + }, + locale: 'en-US', + screen: { density: 2 }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.1.11' }, + campaign: {}, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', + }, + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + integrations: { All: true }, + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + { + destination, + metadata: generateMetadata(2), + message: { + type: 'identify', + sentAt: '2021-01-03T17:02:53.195Z', + userId: 'test', + channel: 'web', + context: { + os: { name: '', version: '' }, + app: { + name: 'RudderLabs JavaScript SDK', + build: '1.0.0', + version: '1.1.11', + namespace: 'com.rudderlabs.javascript', + }, + traits: { + firstName: 'Test', + lastName: 'Rudderlabs', + email: 'test@rudderstack.com', + phone: '+12 345 578 900', + userId: 'test', + title: 'Developer', + organization: 'Rudder', + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + Flagged: false, + Residence: 'Shibuya', + properties: { listId: 'XUepkK', subscribe: true, consent: ['email', 'sms'] }, + }, + locale: 'en-US', + screen: { density: 2 }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.1.11' }, + campaign: {}, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', + }, + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + integrations: { All: true }, + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + destType: 'klaviyo', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + // profile for identify 1 + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'test', + email: 'test_1@rudderstack.com', + first_name: 'Test', + last_name: 'Rudderlabs', + phone_number: '+12 345 578 900', + anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + title: 'Developer', + organization: 'Rudder', + location: { + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + }, + properties: { Flagged: false, Residence: 'Shibuya' }, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + // profile for identify 2 + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'test', + email: 'test@rudderstack.com', + first_name: 'Test', + last_name: 'Rudderlabs', + phone_number: '+12 345 578 900', + anonymous_id: '97c46c81-3140-456d-b2a9-690d70aaca35', + title: 'Developer', + organization: 'Rudder', + location: { + city: 'Tokyo', + region: 'Kanto', + country: 'JP', + zip: '100-0001', + }, + properties: { Flagged: false, Residence: 'Shibuya' }, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + // subscriptiopn for both identify 1 and 2 + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'test_1@rudderstack.com', + phone_number: '+12 345 578 900', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + { + type: 'profile', + attributes: { + email: 'test@rudderstack.com', + phone_number: '+12 345 578 900', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + sms: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + ], + }, + }, + relationships: subscriptionRelations, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(1), generateMetadata(2)], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + }, { id: 'klaviyo-router-150624-test-3', name: 'klaviyo', @@ -1036,7 +1038,7 @@ export const dataV2: RouterTestData[] = [ }, }, { - id: 'klaviyo-router-150624-test-3', + id: 'klaviyo-router-150624-test-4', name: 'klaviyo', description: '150624 -> Retl Router tests to have retl ', scenario: 'Framework', @@ -1118,46 +1120,6 @@ export const dataV2: RouterTestData[] = [ status: 200, body: { output: [ - { - batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: 'https://a.klaviyo.com/api/profile-import', - headers: { - Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', - Accept: 'application/json', - 'Content-Type': 'application/json', - revision: '2024-06-15', - }, - params: {}, - body: { - JSON: { - data: { - type: 'profile', - attributes: { - external_id: 'testklaviyo1@email.com', - anonymous_id: 'anonTestKlaviyo1', - email: 'testklaviyo1@email.com', - first_name: 'Test Klaviyo 1', - properties: {}, - }, - meta: { - patch_properties: {}, - }, - }, - }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, - }, - files: {}, - }, - metadata: [generateMetadata(1)], - batched: false, - statusCode: 200, - destination, - }, { batchedRequest: [ { @@ -1251,6 +1213,48 @@ export const dataV2: RouterTestData[] = [ statusCode: 200, destination, }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-import', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testklaviyo1@email.com', + anonymous_id: 'anonTestKlaviyo1', + email: 'testklaviyo1@email.com', + first_name: 'Test Klaviyo 1', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(1)], + batched: true, + statusCode: 200, + destination, + }, ], }, }, From 5f34cc62b24f8cc1b053342a070f7274be5eb815 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Sat, 27 Jul 2024 11:50:34 +0530 Subject: [PATCH 23/26] chore: address comments --- src/v0/destinations/klaviyo/batchUtil.js | 69 +++++++++---------- src/v0/destinations/klaviyo/batchUtil.test.js | 7 +- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/v0/destinations/klaviyo/batchUtil.js b/src/v0/destinations/klaviyo/batchUtil.js index 0aedfcfd30..71f1528f73 100644 --- a/src/v0/destinations/klaviyo/batchUtil.js +++ b/src/v0/destinations/klaviyo/batchUtil.js @@ -1,9 +1,5 @@ const lodash = require('lodash'); -const { - defaultBatchRequestConfig, - getSuccessRespEvents, - isDefinedAndNotNull, -} = require('../../util'); +const { defaultRequestConfig, getSuccessRespEvents, isDefinedAndNotNull } = require('../../util'); const { JSON_MIME_TYPE } = require('../../util/constant'); const { BASE_ENDPOINT, CONFIG_CATEGORIES, MAX_BATCH_SIZE, revision } = require('./config'); const { buildRequest, getSubscriptionPayload } = require('./util'); @@ -32,53 +28,49 @@ const groupSubscribeResponsesUsingListIdV2 = (subscribeResponseList) => { * subscription= {listId, subscriptionProfileList} */ const generateBatchedSubscriptionRequest = (subscription, destination) => { - const batchEventResponse = defaultBatchRequestConfig(); - // if( !isDefinedAndNotNull(subscription) ) + const subscriptionPayloadResponse = defaultRequestConfig(); // fetching listId from first event as listId is same for all the events const profiles = []; // list of profiles to be subscribed const { listId, subscriptionProfileList } = subscription; subscriptionProfileList.forEach((profileList) => profiles.push(...profileList)); - batchEventResponse.batchedRequest.body.JSON = getSubscriptionPayload(listId, profiles); - batchEventResponse.batchedRequest.endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; - batchEventResponse.batchedRequest.headers = { + subscriptionPayloadResponse.body.JSON = getSubscriptionPayload(listId, profiles); + subscriptionPayloadResponse.endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; + subscriptionPayloadResponse.headers = { Authorization: `Klaviyo-API-Key ${destination.Config.privateApiKey}`, 'Content-Type': JSON_MIME_TYPE, Accept: JSON_MIME_TYPE, revision, }; - return batchEventResponse.batchedRequest; + return subscriptionPayloadResponse; }; /** - * This function updates batchedRequest with profile requests + * This function generates requests using profiles array and returns an array of all these requests * @param {*} profiles - * @param {*} batchedRequest * @param {*} destination */ -const updateBatchEventResponseWithProfileRequests = (profiles, batchedRequest, destination) => { - const profilesRequests = []; - profiles.forEach((profile) => { - profilesRequests.push(buildRequest(profile, destination, CONFIG_CATEGORIES.IDENTIFYV2)); - }); - // we are keeping profiles request prior to subscription ones as first profile creation and then subscription should happen - batchedRequest.unshift(...profilesRequests); +const getProfileRequests = (profiles, destination) => { + const profilePayloadResponses = profiles.map((profile) => + buildRequest(profile, destination, CONFIG_CATEGORIES.IDENTIFYV2), + ); + return profilePayloadResponses; }; /** * this function populates profileSubscriptionAndMetadataArr with respective profiles based upon common metadata * @param {*} profileSubscriptionAndMetadataArr - * @param {*} metadataIndexMap + * @param {*} metaDataIndexMap * @param {*} profiles * @returns updated profileSubscriptionAndMetadataArr obj */ const populateArrWithRespectiveProfileData = ( profileSubscriptionAndMetadataArr, - metadataIndexMap, + metaDataIndexMap, profiles, ) => { - const updatedPSMArr = profileSubscriptionAndMetadataArr; + const updatedPSMArr = lodash.cloneDeep(profileSubscriptionAndMetadataArr); profiles.forEach((profile) => { - const index = metadataIndexMap[profile.metadata.jobId]; + const index = metaDataIndexMap.get(profile.metadata.jobId); if (isDefinedAndNotNull(index)) { // using isDefinedAndNotNull as index can be 0 updatedPSMArr[index].profiles.push(profile.payload); @@ -119,24 +111,31 @@ const buildRequestsForProfileSubscriptionAndMetadataArr = ( profileSubscriptionAndMetadataArr, destination, ) => { - const batchedResponseList = []; - profileSubscriptionAndMetadataArr.forEach((input) => { + const finalResponseList = []; + profileSubscriptionAndMetadataArr.forEach((profileSubscriptionData) => { const batchedRequest = []; - if (input.subscription) { - batchedRequest.push(generateBatchedSubscriptionRequest(input.subscription, destination)); + // we are keeping profiles request prior to subscription ones as first profile creation and then subscription should happen + if (profileSubscriptionData.profiles.length > 0) { + batchedRequest.push(...getProfileRequests(profileSubscriptionData.profiles, destination)); } - updateBatchEventResponseWithProfileRequests(input.profiles, batchedRequest, destination); - batchedResponseList.push( - getSuccessRespEvents(batchedRequest, input.metadataList, destination, true), + + if (profileSubscriptionData.subscription?.subscriptionProfileList?.length > 0) { + batchedRequest.push( + generateBatchedSubscriptionRequest(profileSubscriptionData.subscription, destination), + ); + } + + finalResponseList.push( + getSuccessRespEvents(batchedRequest, profileSubscriptionData.metadataList, destination, true), ); }); - return batchedResponseList; + return finalResponseList; }; const batchRequestV2 = (subscribeRespList, profileRespList, destination) => { const subscribeEventGroups = groupSubscribeResponsesUsingListIdV2(subscribeRespList); let profileSubscriptionAndMetadataArr = []; - const metadataIndexMap = {}; + const metaDataIndexMap = new Map(); Object.keys(subscribeEventGroups).forEach((listId) => { // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] const eventChunks = lodash.chunk(subscribeEventGroups[listId], MAX_BATCH_SIZE); @@ -149,7 +148,7 @@ const batchRequestV2 = (subscribeRespList, profileRespList, destination) => { const jobIdList = metadataList.map((metadata) => metadata.jobId); // push the jobId: index to metadataIndex mapping which let us know the metadata respective payload index position in batched request jobIdList.forEach((jobId) => { - metadataIndexMap[jobId] = index; + metaDataIndexMap.set(jobId, index); }); profileSubscriptionAndMetadataArr.push({ subscription: { subscriptionProfileList, listId }, @@ -160,7 +159,7 @@ const batchRequestV2 = (subscribeRespList, profileRespList, destination) => { }); profileSubscriptionAndMetadataArr = populateArrWithRespectiveProfileData( profileSubscriptionAndMetadataArr, - metadataIndexMap, + metaDataIndexMap, profileRespList, ); /* Till this point I have a profileSubscriptionAndMetadataArr diff --git a/src/v0/destinations/klaviyo/batchUtil.test.js b/src/v0/destinations/klaviyo/batchUtil.test.js index ccaaf840cd..af1afd8670 100644 --- a/src/v0/destinations/klaviyo/batchUtil.test.js +++ b/src/v0/destinations/klaviyo/batchUtil.test.js @@ -46,7 +46,10 @@ describe('populateArrWithRespectiveProfileData', () => { { profiles: [], metadataList: [{ jobId: '1' }], subscriptions: [] }, { profiles: [], metadataList: [{ jobId: '2' }], subscriptions: [] }, ]; - const metadataIndexMap = { 1: 0, 2: 1 }; + const metadataIndexMap = new Map([ + ['1', 0], + ['2', 1], + ]); const profiles = [ { payload: { name: 'John' }, metadata: { jobId: '1' } }, { payload: { name: 'Doe' }, metadata: { jobId: '2' } }, @@ -65,7 +68,7 @@ describe('populateArrWithRespectiveProfileData', () => { // Handles empty profileSubscriptionAndMetadataArr input it('should handle empty profileSubscriptionAndMetadataArr input', () => { const profileSubscriptionAndMetadataArr = []; - const metadataIndexMap = {}; + const metadataIndexMap = new Map(); const profiles = [{ payload: { name: 'John' }, metadata: { jobId: '1' } }]; const result = populateArrWithRespectiveProfileData( From f3e2998fa814d858010d1e6d2cb844687671ad5a Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 30 Jul 2024 08:24:52 +0530 Subject: [PATCH 24/26] fix: value field format and comments resolved --- src/v0/destinations/klaviyo/batchUtil.js | 2 +- .../klaviyo/data/AddedToCart.json | 1 + .../klaviyo/data/KlaviyoTrackV2.json | 15 ++++- .../klaviyo/data/StartedCheckout.json | 3 +- src/v0/util/index.js | 8 +++ .../klaviyo/processor/trackTestDataV2.ts | 61 +++++++++++++++++++ 6 files changed, 87 insertions(+), 3 deletions(-) diff --git a/src/v0/destinations/klaviyo/batchUtil.js b/src/v0/destinations/klaviyo/batchUtil.js index 71f1528f73..3e99e03deb 100644 --- a/src/v0/destinations/klaviyo/batchUtil.js +++ b/src/v0/destinations/klaviyo/batchUtil.js @@ -115,7 +115,7 @@ const buildRequestsForProfileSubscriptionAndMetadataArr = ( profileSubscriptionAndMetadataArr.forEach((profileSubscriptionData) => { const batchedRequest = []; // we are keeping profiles request prior to subscription ones as first profile creation and then subscription should happen - if (profileSubscriptionData.profiles.length > 0) { + if (profileSubscriptionData.profiles?.length > 0) { batchedRequest.push(...getProfileRequests(profileSubscriptionData.profiles, destination)); } diff --git a/src/v0/destinations/klaviyo/data/AddedToCart.json b/src/v0/destinations/klaviyo/data/AddedToCart.json index 302cc79804..d9c89f1e9b 100644 --- a/src/v0/destinations/klaviyo/data/AddedToCart.json +++ b/src/v0/destinations/klaviyo/data/AddedToCart.json @@ -2,6 +2,7 @@ { "destKey": "$value", "sourceKeys": "value", + "metadata":{"type": "isFloat"}, "required": false }, { diff --git a/src/v0/destinations/klaviyo/data/KlaviyoTrackV2.json b/src/v0/destinations/klaviyo/data/KlaviyoTrackV2.json index 8fbf1f4191..a92472d215 100644 --- a/src/v0/destinations/klaviyo/data/KlaviyoTrackV2.json +++ b/src/v0/destinations/klaviyo/data/KlaviyoTrackV2.json @@ -1,7 +1,20 @@ [ { "destKey": "value", - "sourceKeys": ["properties.revenue", "properties.total", "properties.value"] + "sourceKeys": ["properties.revenue", "properties.total", "properties.value"], + "metadata": { + "excludes": [ + "event", + "email", + "phone", + "revenue", + "total", + "value", + "value_currency", + "currency" + ], + "type": "isFloat" + } }, { "destKey": "value_currency", diff --git a/src/v0/destinations/klaviyo/data/StartedCheckout.json b/src/v0/destinations/klaviyo/data/StartedCheckout.json index 1a4691709d..74821af7d2 100644 --- a/src/v0/destinations/klaviyo/data/StartedCheckout.json +++ b/src/v0/destinations/klaviyo/data/StartedCheckout.json @@ -7,7 +7,8 @@ { "destKey": "$value", "sourceKeys": "value", - "required": false + "required": false, + "metadata":{"type": "isFloat"} }, { "destKey": "Categories", diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 12b8d4dd7e..2545b8738b 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -844,6 +844,14 @@ function formatValues(formattedVal, formattingType, typeFormat, integrationsObj) curFormattedVal = formattedVal.trim(); } }, + isFloat: () => { + if (isDefinedAndNotNull(formattedVal)) { + curFormattedVal = parseFloat(formattedVal); + if (Number.isNaN(curFormattedVal)) { + throw new InstrumentationError('Invalid float value'); + } + } + }, }; if (formattingType in formattingFunctions) { diff --git a/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts index d81ee9a4a1..ecea141d38 100644 --- a/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts @@ -229,4 +229,65 @@ export const trackTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'klaviyo-track-150624-test-3', + name: 'klaviyo', + description: '150624 -> Invalid `value` Field Format', + scenario: 'Business', + successCriteria: + 'Response should contain only event payload with vallue field as object and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { enforceEmailAsPrimary: true }), + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'TestEven001', + sentAt: '2021-01-25T16:12:02.048Z', + userId: 'sajal12', + context: { + traits: { + ...commonTraits, + description: 'Sample description', + name: 'Test', + email: 'test@rudderstack.com', + phone: '9112340375', + }, + }, + properties: { ...commonProps, value: { price: 9.99 } }, + anonymousId: '9c6bd77ea9da3e68', + originalTimestamp: '2021-01-25T15:32:56.409Z', + }), + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: 'Invalid float value', + statTags: { + destType: 'KLAVIYO', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + }, + statusCode: 400, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; From a3cc45c8f2ba8e01e1378e7441304c821cb8ed76 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 30 Jul 2024 10:37:51 +0530 Subject: [PATCH 25/26] fix: merge develop fix --- src/v0/util/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/v0/util/index.js b/src/v0/util/index.js index b8e849a429..e7ddfdf095 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -864,6 +864,8 @@ function formatValues(formattedVal, formattingType, typeFormat, integrationsObj) if (Number.isNaN(curFormattedVal)) { throw new InstrumentationError('Invalid float value'); } + } + }, removeSpacesAndDashes: () => { if (typeof formattedVal === 'string') { curFormattedVal = formattedVal.replace(/ /g, '').replace(/-/g, ''); From 1eb83fd92cda00c098adb521d898ac5163be57ad Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 30 Jul 2024 11:05:38 +0530 Subject: [PATCH 26/26] fix: lint errors --- src/v0/destinations/klaviyo/data/AddedToCart.json | 2 +- src/v0/destinations/klaviyo/data/StartedCheckout.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/klaviyo/data/AddedToCart.json b/src/v0/destinations/klaviyo/data/AddedToCart.json index d9c89f1e9b..13ba9e7d47 100644 --- a/src/v0/destinations/klaviyo/data/AddedToCart.json +++ b/src/v0/destinations/klaviyo/data/AddedToCart.json @@ -2,7 +2,7 @@ { "destKey": "$value", "sourceKeys": "value", - "metadata":{"type": "isFloat"}, + "metadata": { "type": "isFloat" }, "required": false }, { diff --git a/src/v0/destinations/klaviyo/data/StartedCheckout.json b/src/v0/destinations/klaviyo/data/StartedCheckout.json index 74821af7d2..10441cb277 100644 --- a/src/v0/destinations/klaviyo/data/StartedCheckout.json +++ b/src/v0/destinations/klaviyo/data/StartedCheckout.json @@ -8,7 +8,7 @@ "destKey": "$value", "sourceKeys": "value", "required": false, - "metadata":{"type": "isFloat"} + "metadata": { "type": "isFloat" } }, { "destKey": "Categories",