From 5388074a60748b2bf4ed498fb5fd2bfc743f9d8e Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 20 Jun 2024 20:42:57 +0530 Subject: [PATCH] feat: update klaviyo api to 2024-06-15 --- src/v0/destinations/klaviyo/config.js | 3 + .../klaviyo/data/KlaviyoIdentifyV2.json | 142 ++++++++++++++++++ src/v0/destinations/klaviyo/transform.js | 97 +++++++++++- src/v0/destinations/klaviyo/util.js | 22 +++ .../destinations/klaviyo/network.ts | 18 --- .../klaviyo/processor/identifyTestData.ts | 107 +++++++++++++ test/integrations/testUtils.ts | 1 + 7 files changed, 370 insertions(+), 20 deletions(-) create mode 100644 src/v0/destinations/klaviyo/data/KlaviyoIdentifyV2.json diff --git a/src/v0/destinations/klaviyo/config.js b/src/v0/destinations/klaviyo/config.js index d8583ab9cb..fea15ac57c 100644 --- a/src/v0/destinations/klaviyo/config.js +++ b/src/v0/destinations/klaviyo/config.js @@ -9,6 +9,7 @@ const MAX_BATCH_SIZE = 100; const CONFIG_CATEGORIES = { IDENTIFY: { name: 'KlaviyoIdentify', apiUrl: '/api/profiles' }, + IDENTIFY_V2: { name: 'KlaviyoIdentifyV2', apiUrl: '/api/profiles' }, SCREEN: { name: 'KlaviyoTrack', apiUrl: '/api/events' }, TRACK: { name: 'KlaviyoTrack', apiUrl: '/api/events' }, GROUP: { name: 'KlaviyoGroup' }, @@ -18,6 +19,7 @@ const CONFIG_CATEGORIES = { ADDED_TO_CART: { name: 'AddedToCart' }, ITEMS: { name: 'Items' }, }; +const { useUpdatedKlaviyoAPI } = process.env; const ecomExclusionKeys = [ 'name', 'product_id', @@ -70,4 +72,5 @@ module.exports = { eventNameMapping, jsonNameMapping, destType, + useUpdatedKlaviyoAPI, }; diff --git a/src/v0/destinations/klaviyo/data/KlaviyoIdentifyV2.json b/src/v0/destinations/klaviyo/data/KlaviyoIdentifyV2.json new file mode 100644 index 0000000000..a78cbd71a4 --- /dev/null +++ b/src/v0/destinations/klaviyo/data/KlaviyoIdentifyV2.json @@ -0,0 +1,142 @@ +[ + { + "destKey": "external_id", + "sourceKeys": "userId", + "required": false, + "sourceFromGenericMap": true + }, + { + "destKey": "anonymous_id", + "sourceKeys": "anonymousId", + "required": false + }, + { + "destKey": "email", + "sourceKeys": "emailOnly", + "required": false, + "sourceFromGenericMap": true + }, + { + "destKey": "first_name", + "sourceKeys": "firstName", + "required": false, + "sourceFromGenericMap": true + }, + { + "destKey": "last_name", + "sourceKeys": "lastName", + "required": false, + "sourceFromGenericMap": true + }, + { + "destKey": "phone_number", + "sourceKeys": "phone", + "required": false, + "sourceFromGenericMap": true + }, + { + "destKey": "title", + "sourceKeys": ["traits.title", "context.traits.title", "properties.title"], + "required": false + }, + { + "destKey": "organization", + "sourceKeys": ["traits.organization", "context.traits.organization", "properties.organization"], + "required": false + }, + { + "destKey": "location.city", + "sourceKeys": [ + "traits.city", + "traits.address.city", + "context.traits.city", + "context.traits.address.city", + "properties.city" + ], + "required": false + }, + { + "destKey": "location.region", + "sourceKeys": [ + "traits.region", + "traits.address.region", + "context.traits.region", + "context.traits.address.region", + "properties.region", + "traits.state", + "traits.address.state", + "context.traits.address.state", + "context.traits.state", + "properties.state" + ], + "required": false + }, + { + "destKey": "location.country", + "sourceKeys": [ + "traits.country", + "traits.address.country", + "context.traits.country", + "context.traits.address.country", + "properties.country" + ], + "required": false + }, + { + "destKey": "location.zip", + "sourceKeys": [ + "traits.zip", + "traits.postalcode", + "traits.postalCode", + "traits.address.zip", + "traits.address.postalcode", + "traits.address.postalCode", + "context.traits.zip", + "context.traits.postalcode", + "context.traits.postalCode", + "context.traits.address.zip", + "context.traits.address.postalcode", + "context.traits.address.postalCode", + "properties.zip", + "properties.postalcode", + "properties.postalCode" + ], + "required": false + }, + { + "destKey": "ip", + "sourceKeys": ["context.ip", "request_ip"], + "required": false + }, + { + "destKey": "_kx", + "sourceKeys": ["traits._kx", "context.traits._kx"], + "required": false + }, + { + "destKey": "location.timezone", + "sourceKeys": ["traits.timezone", "context.traits.timezone", "properties.timezone"], + "required": false + }, + { + "destKey": "latitude", + "sourceKeys": ["latitude", "context.address.latitude", "context.location.latitude"], + "required": false + }, + { + "destKey": "longitude", + "sourceKeys": ["longitude", "context.address.longitude", "context.location.longitude"], + "required": false + }, + { + "destKey": "location.address1", + "sourceKeys": [ + "traits.street", + "traits.address.street", + "context.traits.street", + "context.traits.address.street", + "properties.street" + ], + "required": false + } +] diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js index 09e75919f9..4fd1c74eb0 100644 --- a/src/v0/destinations/klaviyo/transform.js +++ b/src/v0/destinations/klaviyo/transform.js @@ -2,6 +2,7 @@ /* eslint-disable no-underscore-dangle */ /* eslint-disable array-callback-return */ const get = require('get-value'); +const set = require('set-value'); const { ConfigurationError, InstrumentationError } = require('@rudderstack/integrations-lib'); const { EventType, WhiteListedTraits, MappedToDestinationKey } = require('../../../constants'); const { @@ -12,6 +13,7 @@ const { ecomEvents, eventNameMapping, jsonNameMapping, + useUpdatedKlaviyoAPI, } = require('./config'); const { createCustomerProperties, @@ -20,6 +22,7 @@ const { batchSubscribeEvents, getIdFromNewOrExistingProfile, profileUpdateResponseBuilder, + createCustomerPropertiesV2, } = require('./util'); const { defaultRequestConfig, @@ -35,9 +38,89 @@ const { handleRtTfSingleEventError, flattenJson, isNewStatusCodesAccepted, + getDestinationExternalID, + isDefinedAndNotNullAndNotEmpty, } = require('../../util'); const { JSON_MIME_TYPE, HTTP_STATUS_CODES } = require('../../util/constant'); +/** + * Identify function to handle request for Klaviyo version '2024-06-15' + * The function is used to create/update new (same endpoint and no intermediate API calls) users and also for adding/subscribing + * users to the list. + * DOCS: 1. https://developers.klaviyo.com/en/reference/create_or_update_profile + * 2. https://developers.klaviyo.com/en/v2023-02-22/reference/subscribe_profiles + * @param {*} message + * @param {*} category + * @param {*} destination + * @param {*} reqMetadata + * @returns + */ +const identifyRequestHandlerV2 = (message, category, destination) => { + // If listId property is present try to subscribe/member user in list + const { privateApiKey, listId, flattenProperties } = destination.Config; + const mappedToDestination = get(message, MappedToDestinationKey); + if (mappedToDestination) { + addExternalIdToTraits(message); + adduserIdFromExternalId(message); + } + const traitsInfo = getFieldValueFromMessage(message, 'traits'); + let propertyPayload = constructPayload(message, MAPPING_CONFIG[category.name]); + // Extract other K-V property from traits about user custom properties + let customPropertyPayload = {}; + customPropertyPayload = extractCustomFields( + message, + customPropertyPayload, + ['traits', 'context.traits'], + [...WhiteListedTraits, '_kx'], + ); + + propertyPayload = removeUndefinedAndNullValues(propertyPayload); + const data = { + type: 'profile', + attributes: { + ...propertyPayload, + properties: removeUndefinedAndNullValues(customPropertyPayload), + }, + }; + // if flattenProperties is enabled from UI, flatten the user properties + data.attributes.properties = flattenProperties + ? flattenJson(data.attributes.properties, '.', 'normal', false) + : data.attributes.properties; + // external id -> Klaviyo generated id for every profile + const externalId = getDestinationExternalID(message, 'klaviyoExternalId'); + if (isDefinedAndNotNullAndNotEmpty(externalId)) { + set(data, 'id', externalId); + } + if (isEmptyObject(data.attributes.properties)) { + delete data.attributes.properties; + } + const payload = { + data: removeUndefinedAndNullValues(data), + }; + const endpoint = `${BASE_ENDPOINT}${category.apiUrl}`; + const requestOptions = { + headers: { + Authorization: `Klaviyo-API-Key ${privateApiKey}`, + Accept: JSON_MIME_TYPE, + 'Content-Type': JSON_MIME_TYPE, + revision: '2024-06-15', + }, + }; + + const profileRequest = defaultRequestConfig(); + profileRequest.endpoint = endpoint; + profileRequest.body.JSON = payload; + profileRequest.headers = requestOptions.headers; + + let subscriptionRequest; + // check if user wants to subscribe profile or not and listId is present or not + if (traitsInfo?.properties?.subscribe && (traitsInfo.properties?.listId || listId)) { + subscriptionRequest = subscribeUserToList(message, traitsInfo, destination); + return [profileRequest, subscriptionRequest]; + } + + return profileRequest; +}; /** * Main Identify request handler func * The function is used to create/update new users and also for adding/subscribing @@ -157,7 +240,7 @@ const identifyRequestHandler = async ( const trackRequestHandler = (message, category, destination) => { const payload = {}; - const { privateApiKey, flattenProperties } = destination.Config; + const { privateApiKey, flattenProperties, useUpdatedKlaviyo } = destination.Config; let event = get(message, 'event'); if (event && typeof event !== 'string') { throw new InstrumentationError('Event type should be a string'); @@ -225,8 +308,12 @@ const trackRequestHandler = (message, category, destination) => { ? flattenJson(attributes.properties, '.', 'normal', false) : attributes.properties; // Map user properties to profile object + const customerProp = + useUpdatedKlaviyoAPI || useUpdatedKlaviyo + ? createCustomerProperties(message, destination.Config) + : createCustomerPropertiesV2(message, destination.Config); attributes.profile = { - ...createCustomerProperties(message, destination.Config), + ...customerProp, ...populateCustomFieldsFromTraits(message), }; @@ -289,6 +376,12 @@ const processEvent = async (event, reqMetadata) => { let response; switch (messageType) { case EventType.IDENTIFY: + // checking if we want to use the updated klaviyo api for identify + if (useUpdatedKlaviyoAPI || destination.Config?.useUpdatedKlaviyo) { + category = CONFIG_CATEGORIES.IDENTIFY_V2; + response = identifyRequestHandlerV2(message, category, destination); + break; + } category = CONFIG_CATEGORIES.IDENTIFY; response = await identifyRequestHandler( { message, category, destination, metadata }, diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 4db59cfb05..3f1e3f7d67 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -208,6 +208,27 @@ const createCustomerProperties = (message, Config) => { customerProperties = removeUndefinedAndNullValues(customerProperties); return customerProperties; }; +// This function is used for creating and returning customer properties using mapping json +const createCustomerPropertiesV2 = (message, Config) => { + const { enforceEmailAsPrimary } = Config; + let customerProperties = constructPayload( + message, + MAPPING_CONFIG[CONFIG_CATEGORIES.PROFILE.name], + ); + if (!enforceEmailAsPrimary) { + customerProperties.$id = getFieldValueFromMessage(message, 'userIdOnly'); + } else { + if (!customerProperties.$email && !customerProperties.$phone_number) { + throw new InstrumentationError('None of email and phone are present in the payload'); + } + customerProperties = { + ...customerProperties, + _id: getFieldValueFromMessage(message, 'userIdOnly'), + }; + } + customerProperties = removeUndefinedAndNullValues(customerProperties); + return customerProperties; +}; const populateCustomFieldsFromTraits = (message) => { // Extract other K-V property from traits about user custom properties @@ -327,6 +348,7 @@ const batchSubscribeEvents = (subscribeRespList) => { module.exports = { subscribeUserToList, createCustomerProperties, + createCustomerPropertiesV2, populateCustomFieldsFromTraits, generateBatchedPaylaodForArray, batchSubscribeEvents, diff --git a/test/integrations/destinations/klaviyo/network.ts b/test/integrations/destinations/klaviyo/network.ts index d76d235c6f..1b525ee9e6 100644 --- a/test/integrations/destinations/klaviyo/network.ts +++ b/test/integrations/destinations/klaviyo/network.ts @@ -1,22 +1,4 @@ export const networkCallsData = [ - { - httpReq: { - url: 'https://a.klaviyo.com/api/v2/list/XUepkK/subscribe', - method: 'GET', - }, - httpRes: { - status: 200, - }, - }, - { - httpReq: { - url: 'https://a.klaviyo.com/api/v2/list/XUepkK/members', - method: 'GET', - }, - httpRes: { - status: 200, - }, - }, { httpReq: { url: 'https://a.klaviyo.com/api/profiles', diff --git a/test/integrations/destinations/klaviyo/processor/identifyTestData.ts b/test/integrations/destinations/klaviyo/processor/identifyTestData.ts index 0dd4751133..20544cae42 100644 --- a/test/integrations/destinations/klaviyo/processor/identifyTestData.ts +++ b/test/integrations/destinations/klaviyo/processor/identifyTestData.ts @@ -88,6 +88,10 @@ const commonOutputSubscriptionProps = { }, ], }; +const location = { + longitude: '0.1.2.2', + latitude: '0.1.1.1', +}; const commonOutputHeaders = { Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', @@ -95,7 +99,14 @@ const commonOutputHeaders = { Accept: 'application/json', revision: '2023-02-22', }; +const commonOutputHeadersUpdated = { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + 'Content-Type': 'application/json', + Accept: 'application/json', + revision: '2024-06-15', +}; +const updatedEndpoint = 'https://a.klaviyo.com/api/profiles'; const anonymousId = '97c46c81-3140-456d-b2a9-690d70aaca35'; const userId = 'user@1'; const sentAt = '2021-01-03T17:02:53.195Z'; @@ -590,4 +601,100 @@ export const identifyData: ProcessorTestData[] = [ }, }, }, + { + id: 'klaviyo-identify-test-8', + name: 'klaviyo', + description: + 'Updated Identify call for with flattenProperties enabled in destination config and subscription also present', + scenario: 'Business', + successCriteria: + 'The profile Create or update response should contain the flattened properties of the friend object and subscription request payload as well', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { + flattenProperties: true, + useUpdatedKlaviyo: true, + }), + message: generateSimplifiedIdentifyPayload({ + sentAt, + userId, + context: { + traits: { + ...commonTraits2, + _kx: 'encrytped number', + friend: { + names: { + first: 'Alice', + last: 'Smith', + }, + age: 25, + }, + }, + ip: '0.0.0.0', + location, + }, + anonymousId, + originalTimestamp, + }), + metadata: generateMetadata(2), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + method: 'POST', + endpoint: updatedEndpoint, + headers: commonOutputHeadersUpdated, + JSON: { + data: { + type: 'profile', + attributes: { + ...commonOutputUserProps2, + ip: '0.0.0.0', + anonymous_id: anonymousId, + _kx: 'encrytped number', + properties: { + ...commonOutputUserProps2.properties, + 'friend.age': 25, + 'friend.names.first': 'Alice', + 'friend.names.last': 'Smith', + }, + }, + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(2), + }, + { + output: transformResultBuilder({ + userId: '', + method: 'POST', + endpoint: subscribeEndpoint, + headers: commonOutputHeaders, + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: commonOutputSubscriptionProps, + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(2), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/testUtils.ts b/test/integrations/testUtils.ts index a6f0720e37..eb8f4197ee 100644 --- a/test/integrations/testUtils.ts +++ b/test/integrations/testUtils.ts @@ -187,6 +187,7 @@ export const generateSimplifiedIdentifyPayload: any = (parametersOverride: any) context: { externalId: parametersOverride.context.externalId, traits: parametersOverride.context.traits, + ip: parametersOverride.context.ip, }, anonymousId: parametersOverride.anonymousId || 'default-anonymousId', originalTimestamp: parametersOverride.originalTimestamp || '2021-01-03T17:02:53.193Z',