From f7d04411a498191c832b75bbf1fb15e0e18be3b1 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Wed, 13 Sep 2023 13:09:24 +0530 Subject: [PATCH] feat: onboard launchdarkly audience (#2529) * feat: onboard launchdarkly audience * docs: add comment * test: add testcases * refactor: update identifier logic * feat: add context kind mapping * fix: use removeUndefinedNullEmptyExclBoolInt util * chore: stop sending audience url * chore: remove extra argument in prepareIdentifiersList * fix: endpoint * fix: testcases * feat: add batching support * chore: add unit test cases --- .../launchdarkly_audience/config.js | 15 + .../launchdarkly_audience/procWorkflow.yaml | 67 +++ .../launchdarkly_audience/utils.js | 87 +++ .../launchdarkly_audience/utils.test.js | 136 +++++ .../__tests__/data/launchdarkly_audience.json | 495 ++++++++++++++++++ .../launchdarkly_audience-cdk.test.ts | 42 ++ 6 files changed, 842 insertions(+) create mode 100644 src/cdk/v2/destinations/launchdarkly_audience/config.js create mode 100644 src/cdk/v2/destinations/launchdarkly_audience/procWorkflow.yaml create mode 100644 src/cdk/v2/destinations/launchdarkly_audience/utils.js create mode 100644 src/cdk/v2/destinations/launchdarkly_audience/utils.test.js create mode 100644 test/__tests__/data/launchdarkly_audience.json create mode 100644 test/__tests__/launchdarkly_audience-cdk.test.ts diff --git a/src/cdk/v2/destinations/launchdarkly_audience/config.js b/src/cdk/v2/destinations/launchdarkly_audience/config.js new file mode 100644 index 0000000000..a1ec48a43c --- /dev/null +++ b/src/cdk/v2/destinations/launchdarkly_audience/config.js @@ -0,0 +1,15 @@ +const SUPPORTED_EVENT_TYPE = 'audiencelist'; +const ACTION_TYPES = ['add', 'remove']; +const IDENTIFIER_KEY = 'identifier'; +// ref:- https://docs.launchdarkly.com/guides/integrations/build-synced-segments?q=othercapabilities#manifest-details +// ref:- https://docs.launchdarkly.com/home/segments#targeting-users-in-segments +const ENDPOINT = 'https://app.launchdarkly.com/api/v2/segment-targets/rudderstack'; +const MAX_IDENTIFIERS = 1000; + +module.exports = { + SUPPORTED_EVENT_TYPE, + ACTION_TYPES, + IDENTIFIER_KEY, + ENDPOINT, + MAX_IDENTIFIERS, +}; diff --git a/src/cdk/v2/destinations/launchdarkly_audience/procWorkflow.yaml b/src/cdk/v2/destinations/launchdarkly_audience/procWorkflow.yaml new file mode 100644 index 0000000000..48aad2bb79 --- /dev/null +++ b/src/cdk/v2/destinations/launchdarkly_audience/procWorkflow.yaml @@ -0,0 +1,67 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + exportAll: true + - name: defaultRequestConfig + path: ../../../../v0/util + - name: removeUndefinedNullEmptyExclBoolInt + path: ../../../../v0/util + - path: ./config + exportAll: true + - path: ./utils + exportAll: true + +steps: + - name: validateInput + template: | + let messageType = .message.type; + $.assertConfig(.destination.Config.audienceId, "Audience Id is not present. Aborting"); + $.assertConfig(.destination.Config.audienceName, "Audience Name is not present. Aborting"); + $.assertConfig(.destination.Config.accessToken, "Access Token is not present. Aborting"); + $.assertConfig(.destination.Config.clientSideId, "Launch Darkly Client Side is not present. Aborting"); + $.assert(.message.type, "Message Type is not present. Aborting message."); + $.assert(.message.type.toLowerCase() === $.SUPPORTED_EVENT_TYPE, "Event type " + .message.type.toLowerCase() + " is not supported. Aborting message."); + $.assert(.message.properties, "Message properties is not present. Aborting message."); + $.assert(.message.properties.listData, "`listData` is not present inside properties. Aborting message."); + $.assert($.containsAll(Object.keys(.message.properties.listData), $.ACTION_TYPES), "Unsupported action type. Aborting message.") + $.assert(Object.keys(.message.properties.listData).length > 0, "`listData` is empty. Aborting message.") + + - name: batchIdentifiersList + description: batch identifiers list + template: | + const batchedList = $.batchIdentifiersList(.message.properties.listData); + $.assert(batchedList.length > 0, "`listData` is empty. Aborting message."); + batchedList; + + - name: prepareBasePayload + template: | + const payload = { + environmentId: .destination.Config.clientSideId, + cohortId: .destination.Config.audienceId, + cohortName: .destination.Config.audienceName, + contextKind: .destination.Config.audienceType + }; + $.removeUndefinedNullEmptyExclBoolInt(payload); + + - name: buildResponseForProcessTransformation + description: build multiplexed response depending upon batch size + template: | + $.outputs.batchIdentifiersList.().({ + "body": { + "JSON": {...$.outputs.prepareBasePayload, listData: .}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": {{$.ENDPOINT}}, + "headers": { + "Authorization": ^.destination.Config.accessToken, + "Content-Type": "application/json" + }, + "params": {}, + "files": {} + })[] diff --git a/src/cdk/v2/destinations/launchdarkly_audience/utils.js b/src/cdk/v2/destinations/launchdarkly_audience/utils.js new file mode 100644 index 0000000000..5bbcdfb6a3 --- /dev/null +++ b/src/cdk/v2/destinations/launchdarkly_audience/utils.js @@ -0,0 +1,87 @@ +const _ = require('lodash'); +const { ACTION_TYPES, IDENTIFIER_KEY, MAX_IDENTIFIERS } = require('./config'); + +/** + * Prepares a list of identifiers based on the provided data and identifier key. + * @param {*} listData The data containing lists of members to be added or removed from the audience. + * @returns + * { + "add": [ + { + "id": "test@gmail.com" + } + ], + "remove": [] + } + */ +const prepareIdentifiersList = (listData) => { + const list = {}; + const processList = (actionData) => + actionData + .filter((member) => member.hasOwnProperty(IDENTIFIER_KEY) && member[IDENTIFIER_KEY]) + .map((member) => ({ id: member[IDENTIFIER_KEY] })); + ACTION_TYPES.forEach((action) => { + list[action] = listData?.[action] ? processList(listData[action]) : []; + }); + return list; +}; + +/** + * Batch the identifiers list based on the maximum number of identifiers allowed per request. + * @param {*} listData The data containing lists of members to be added or removed from the audience. + * @returns + * For MAX_IDENTIFIERS = 2 + * [ + { + "add": [ + { + "id": "test1@gmail.com" + } + ], + "remove": [ + { + "id": "test2@gmail.com" + } + ] + }, + { + "add": [ + { + "id": "test3@gmail.com" + } + ], + "remove": [] + } + ] + */ +const batchIdentifiersList = (listData) => { + const audienceList = prepareIdentifiersList(listData); + const combinedList = [ + ...audienceList.add.map((item) => ({ ...item, type: 'add' })), + ...audienceList.remove.map((item) => ({ ...item, type: 'remove' })), + ]; + + const chunkedData = _.chunk(combinedList, MAX_IDENTIFIERS); + + // Group the chunks by action type (add/remove) + const groupedData = chunkedData.map((chunk) => { + const groupedChunk = { + add: [], + remove: [], + }; + + chunk.forEach((item) => { + if (item.type === 'add') { + groupedChunk.add.push({ id: item.id }); + } else if (item.type === 'remove') { + groupedChunk.remove.push({ id: item.id }); + } + }); + + return groupedChunk; + }); + + return groupedData; +}; + +module.exports = { prepareIdentifiersList, batchIdentifiersList }; diff --git a/src/cdk/v2/destinations/launchdarkly_audience/utils.test.js b/src/cdk/v2/destinations/launchdarkly_audience/utils.test.js new file mode 100644 index 0000000000..8c06c99076 --- /dev/null +++ b/src/cdk/v2/destinations/launchdarkly_audience/utils.test.js @@ -0,0 +1,136 @@ +const { prepareIdentifiersList, batchIdentifiersList } = require('./utils'); + +jest.mock(`./config`, () => { + const originalConfig = jest.requireActual(`./config`); + return { + ...originalConfig, + MAX_IDENTIFIERS: 2, + }; +}); + +describe('Unit test cases for prepareIdentifiersList', () => { + it('should return an object with empty "add" and "remove" properties when no data is provided', () => { + const result = prepareIdentifiersList({}); + expect(result).toEqual({ add: [], remove: [] }); + }); + + it('should handle null input and return an object with empty "add" and "remove" identifiers list', () => { + const result = prepareIdentifiersList(null); + expect(result).toEqual({ add: [], remove: [] }); + }); + + it('should handle undefined input and return an object with empty "add" and "remove" identifiers list', () => { + const result = prepareIdentifiersList(undefined); + expect(result).toEqual({ add: [], remove: [] }); + }); + + it('should handle input with missing "add" or "remove" identifiers list and return an object with empty "add" and "remove" identifiers list', () => { + const result = prepareIdentifiersList({ add: [], remove: undefined }); + expect(result).toEqual({ add: [], remove: [] }); + }); + + it('should handle input with empty "add" or "remove" identifiers list and return an object with empty "add" and "remove" identifiers list', () => { + const result = prepareIdentifiersList({ add: [], remove: [] }); + expect(result).toEqual({ add: [], remove: [] }); + }); + + it('should handle input with non empty "add" or "remove" identifiers list and return an object non empty "add" and "remove" identifiers list', () => { + const result = prepareIdentifiersList({ + add: [{ identifier: 'test1@email.com' }, { identifier: 'test2@email.com' }], + remove: [{ identifier: 'test3@email.com' }], + }); + expect(result).toEqual({ + add: [{ id: 'test1@email.com' }, { id: 'test2@email.com' }], + remove: [{ id: 'test3@email.com' }], + }); + }); +}); + +describe('Unit test cases for batchIdentifiersList', () => { + it('should correctly batch a list containing both "add" and "remove" actions', () => { + const listData = { + add: [ + { identifier: 'test1@email.com' }, + { identifier: 'test2@email.com' }, + { identifier: 'test3@email.com' }, + ], + remove: [{ identifier: 'test4@email.com' }, { identifier: 'test5@email.com' }], + }; + + const expectedOutput = [ + { + add: [{ id: 'test1@email.com' }, { id: 'test2@email.com' }], + remove: [], + }, + { + add: [{ id: 'test3@email.com' }], + remove: [{ id: 'test4@email.com' }], + }, + { + add: [], + remove: [{ id: 'test5@email.com' }], + }, + ]; + + expect(batchIdentifiersList(listData)).toEqual(expectedOutput); + }); + + it('should correctly batch a list containing only "add" actions', () => { + const listData = { + add: [ + { identifier: 'test1@email.com' }, + { identifier: 'test2@email.com' }, + { identifier: 'test3@email.com' }, + ], + remove: [], + }; + + const expectedOutput = [ + { + add: [{ id: 'test1@email.com' }, { id: 'test2@email.com' }], + remove: [], + }, + { + add: [{ id: 'test3@email.com' }], + remove: [], + }, + ]; + + expect(batchIdentifiersList(listData)).toEqual(expectedOutput); + }); + + it('should correctly batch a list containing only "remove" actions', () => { + const listData = { + add: [], + remove: [ + { identifier: 'test1@email.com' }, + { identifier: 'test2@email.com' }, + { identifier: 'test3@email.com' }, + ], + }; + + const expectedOutput = [ + { + add: [], + remove: [{ id: 'test1@email.com' }, { id: 'test2@email.com' }], + }, + { + add: [], + remove: [{ id: 'test3@email.com' }], + }, + ]; + + expect(batchIdentifiersList(listData)).toEqual(expectedOutput); + }); + + it('should return an empty list for empty input list data', () => { + const listData = { + add: [{ identifier: '' }], + remove: [], + }; + + const expectedOutput = []; + + expect(batchIdentifiersList(listData)).toEqual(expectedOutput); + }); +}); diff --git a/test/__tests__/data/launchdarkly_audience.json b/test/__tests__/data/launchdarkly_audience.json new file mode 100644 index 0000000000..2f348e01fa --- /dev/null +++ b/test/__tests__/data/launchdarkly_audience.json @@ -0,0 +1,495 @@ +[ + { + "description": "Unsupported event type", + "input": { + "message": { + "userId": "user123", + "type": "abc", + "properties": { + "listData": { + "add": [ + { + "identifier": "alex@email.com" + }, + { + "identifier": "ryan@email.com" + }, + { + "identifier": "van@email.com" + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "destination": { + "Config": { + "audienceId": "test-audienceId", + "audienceName": "test-audienceName", + "accessToken": "test-accessToken", + "clientSideId": "test-clientSideId" + } + } + }, + "output": { + "error": "Event type abc is not supported. Aborting message." + } + }, + { + "description": "List data is not passed", + "input": { + "message": { + "userId": "user123", + "type": "audiencelist", + "properties": {}, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "destination": { + "Config": { + "audienceId": "test-audienceId", + "audienceName": "test-audienceName", + "accessToken": "test-accessToken", + "clientSideId": "test-clientSideId" + } + } + }, + "output": { + "error": "`listData` is not present inside properties. Aborting message." + } + }, + { + "description": "List data is empty", + "input": { + "message": { + "userId": "user123", + "type": "audiencelist", + "properties": { + "listData": {} + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "destination": { + "Config": { + "audienceId": "test-audienceId", + "audienceName": "test-audienceName", + "accessToken": "test-accessToken", + "clientSideId": "test-clientSideId" + } + } + }, + "output": { + "error": "`listData` is empty. Aborting message." + } + }, + { + "description": "List data is empty", + "input": { + "message": { + "userId": "user123", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "identifier": "" + }, + { + "identifier": "" + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "destination": { + "Config": { + "audienceId": "test-audienceId", + "audienceName": "test-audienceName", + "accessToken": "test-accessToken", + "clientSideId": "test-clientSideId" + } + } + }, + "output": { + "error": "`listData` is empty. Aborting message." + } + }, + { + "description": "Unsupported action type", + "input": { + "message": { + "userId": "user123", + "type": "audiencelist", + "properties": { + "listData": { + "update": [ + { + "identifier": "alex@email.com" + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "destination": { + "Config": { + "audienceId": "test-audienceId", + "audienceName": "test-audienceName", + "accessToken": "test-accessToken", + "clientSideId": "test-clientSideId" + } + } + }, + "output": { + "error": "Unsupported action type. Aborting message." + } + }, + { + "description": "Add members to the audience list", + "input": { + "message": { + "userId": "user123", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "identifier": "alex@email.com" + }, + { + "identifier": "ryan@email.com" + }, + { + "identifier": "van@email.com" + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "destination": { + "Config": { + "audienceId": "test-audienceId", + "audienceName": "test-audienceName", + "accessToken": "test-accessToken", + "clientSideId": "test-clientSideId" + } + } + }, + "output": [ + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://app.launchdarkly.com/api/v2/segment-targets/rudderstack", + "headers": { + "Authorization": "test-accessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "environmentId": "test-clientSideId", + "cohortId": "test-audienceId", + "cohortName": "test-audienceName", + "listData": { + "add": [ + { + "id": "alex@email.com" + }, + { + "id": "ryan@email.com" + } + ], + "remove": [] + } + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + }, + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://app.launchdarkly.com/api/v2/segment-targets/rudderstack", + "headers": { + "Authorization": "test-accessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "environmentId": "test-clientSideId", + "cohortId": "test-audienceId", + "cohortName": "test-audienceName", + "listData": { + "add": [ + { + "id": "van@email.com" + } + ], + "remove": [] + } + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + ] + }, + { + "description": "Remove members from the audience list", + "input": { + "message": { + "userId": "user123", + "type": "audiencelist", + "properties": { + "listData": { + "remove": [ + { + "identifier": "alex@email.com" + }, + { + "identifier": "ryan@email.com" + }, + { + "identifier": "van@email.com" + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "destination": { + "Config": { + "audienceId": "test-audienceId", + "audienceName": "test-audienceName", + "accessToken": "test-accessToken", + "clientSideId": "test-clientSideId" + } + } + }, + "output": [ + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://app.launchdarkly.com/api/v2/segment-targets/rudderstack", + "headers": { + "Authorization": "test-accessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "environmentId": "test-clientSideId", + "cohortId": "test-audienceId", + "cohortName": "test-audienceName", + "listData": { + "remove": [ + { + "id": "alex@email.com" + }, + { + "id": "ryan@email.com" + } + ], + "add": [] + } + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + }, + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://app.launchdarkly.com/api/v2/segment-targets/rudderstack", + "headers": { + "Authorization": "test-accessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "environmentId": "test-clientSideId", + "cohortId": "test-audienceId", + "cohortName": "test-audienceName", + "listData": { + "remove": [ + { + "id": "van@email.com" + } + ], + "add": [] + } + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + ] + }, + { + "description": "Add/Remove members", + "input": { + "message": { + "userId": "user123", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "identifier": "alex@email.com" + }, + { + "userId": "user1" + } + ], + "remove": [ + { + "identifier": "ryan@email.com" + }, + { + "identifier": "van@email.com" + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "destination": { + "Config": { + "audienceId": "test-audienceId", + "audienceName": "test-audienceName", + "accessToken": "test-accessToken", + "clientSideId": "test-clientSideId" + } + } + }, + "output": [ + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://app.launchdarkly.com/api/v2/segment-targets/rudderstack", + "headers": { + "Authorization": "test-accessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "environmentId": "test-clientSideId", + "cohortId": "test-audienceId", + "cohortName": "test-audienceName", + "listData": { + "add": [ + { + "id": "alex@email.com" + } + ], + "remove": [ + { + "id": "ryan@email.com" + } + ] + } + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + }, + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://app.launchdarkly.com/api/v2/segment-targets/rudderstack", + "headers": { + "Authorization": "test-accessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "environmentId": "test-clientSideId", + "cohortId": "test-audienceId", + "cohortName": "test-audienceName", + "listData": { + "add": [], + "remove": [ + { + "id": "van@email.com" + } + ] + } + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + ] + } +] diff --git a/test/__tests__/launchdarkly_audience-cdk.test.ts b/test/__tests__/launchdarkly_audience-cdk.test.ts new file mode 100644 index 0000000000..419b59fbd1 --- /dev/null +++ b/test/__tests__/launchdarkly_audience-cdk.test.ts @@ -0,0 +1,42 @@ +import fs from 'fs'; +import path from 'path'; +import { processCdkV2Workflow } from '../../src/cdk/v2/handler'; +import tags from '../../src/v0/util/tags'; + +const integration = 'launchdarkly_audience'; +const destName = 'LaunchDarkly Audience'; + +// Processor Test files +const testDataFile = fs.readFileSync(path.resolve(__dirname, `./data/${integration}.json`), { + encoding: 'utf8', +}); +const testData = JSON.parse(testDataFile); + +jest.mock(`../../src/cdk/v2/destinations/launchdarkly_audience/config`, () => { + const originalConfig = jest.requireActual( + `../../src/cdk/v2/destinations/launchdarkly_audience/config`, + ); + return { + ...originalConfig, + MAX_IDENTIFIERS: 2, + }; +}); + +describe(`${destName} Tests`, () => { + describe('Processor Tests', () => { + testData.forEach((dataPoint, index) => { + it(`${destName} - payload: ${index}`, async () => { + try { + const output = await processCdkV2Workflow( + integration, + dataPoint.input, + tags.FEATURES.PROCESSOR, + ); + expect(output).toEqual(dataPoint.output); + } catch (error: any) { + expect(error.message).toEqual(dataPoint.output.error); + } + }); + }); + }); +});