From 2f30c56af62e983d09b5d4f2da9a0ba22f5c1612 Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:45:37 +0530 Subject: [PATCH 1/4] fix: ninetailed: remove page support (#3218) --- src/cdk/v2/destinations/ninetailed/config.js | 4 - .../ninetailed/data/pageMapping.json | 7 -- .../destinations/ninetailed/procWorkflow.yaml | 2 +- src/cdk/v2/destinations/ninetailed/utils.js | 6 - .../destinations/ninetailed/processor/data.ts | 3 +- .../destinations/ninetailed/processor/page.ts | 108 ------------------ .../destinations/ninetailed/router/data.ts | 48 ++------ 7 files changed, 14 insertions(+), 164 deletions(-) delete mode 100644 src/cdk/v2/destinations/ninetailed/data/pageMapping.json delete mode 100644 test/integrations/destinations/ninetailed/processor/page.ts diff --git a/src/cdk/v2/destinations/ninetailed/config.js b/src/cdk/v2/destinations/ninetailed/config.js index c38496a415..a59b2a1671 100644 --- a/src/cdk/v2/destinations/ninetailed/config.js +++ b/src/cdk/v2/destinations/ninetailed/config.js @@ -17,10 +17,6 @@ const ConfigCategories = { type: 'identify', name: 'identifyMapping', }, - PAGE: { - type: 'page', - name: 'pageMapping', - }, }; // MAX_BATCH_SIZE : // Maximum number of events to send in a single batch diff --git a/src/cdk/v2/destinations/ninetailed/data/pageMapping.json b/src/cdk/v2/destinations/ninetailed/data/pageMapping.json deleted file mode 100644 index 80ec2f58f1..0000000000 --- a/src/cdk/v2/destinations/ninetailed/data/pageMapping.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "sourceKeys": "properties", - "required": true, - "destKey": "properties" - } -] diff --git a/src/cdk/v2/destinations/ninetailed/procWorkflow.yaml b/src/cdk/v2/destinations/ninetailed/procWorkflow.yaml index 6f5056ce10..383b850a4d 100644 --- a/src/cdk/v2/destinations/ninetailed/procWorkflow.yaml +++ b/src/cdk/v2/destinations/ninetailed/procWorkflow.yaml @@ -16,7 +16,7 @@ steps: template: | let messageType = $.outputs.messageType; $.assert(messageType, "message Type is not present. Aborting"); - $.assert(messageType in {{$.EventType.([.TRACK,.IDENTIFY,.PAGE])}}, "message type " + messageType + " is not supported"); + $.assert(messageType in {{$.EventType.([.TRACK,.IDENTIFY])}}, "message type " + messageType + " is not supported"); $.assertConfig(.destination.Config.organisationId, "Organisation ID is not present. Aborting"); $.assertConfig(.destination.Config.environment, "Environment is not present. Aborting"); - name: preparePayload diff --git a/src/cdk/v2/destinations/ninetailed/utils.js b/src/cdk/v2/destinations/ninetailed/utils.js index b716422a0e..47b27b3b9d 100644 --- a/src/cdk/v2/destinations/ninetailed/utils.js +++ b/src/cdk/v2/destinations/ninetailed/utils.js @@ -31,12 +31,6 @@ const constructFullPayload = (message) => { config.mappingConfig[config.ConfigCategories.IDENTIFY.name], ); break; - case 'page': - typeSpecifcPayload = constructPayload( - message, - config.mappingConfig[config.ConfigCategories.PAGE.name], - ); - break; default: break; } diff --git a/test/integrations/destinations/ninetailed/processor/data.ts b/test/integrations/destinations/ninetailed/processor/data.ts index 4e5fa72365..9d3cd217cd 100644 --- a/test/integrations/destinations/ninetailed/processor/data.ts +++ b/test/integrations/destinations/ninetailed/processor/data.ts @@ -1,5 +1,4 @@ import { validationFailures } from './validation'; import { track } from './track'; -import { page } from './page'; import { identify } from './identify'; -export const data = [...identify, ...page, ...track, ...validationFailures]; +export const data = [...identify, ...track, ...validationFailures]; diff --git a/test/integrations/destinations/ninetailed/processor/page.ts b/test/integrations/destinations/ninetailed/processor/page.ts deleted file mode 100644 index 93a086ceea..0000000000 --- a/test/integrations/destinations/ninetailed/processor/page.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { destination, context, commonProperties, metadata } from '../commonConfig'; -import { transformResultBuilder } from '../../../testUtils'; -export const page = [ - { - id: 'ninetailed-test-page-success-1', - name: 'ninetailed', - description: 'page call with all mappings available', - scenario: 'Framework+Buisness', - successCriteria: 'Response should contain all the mappings and status code should be 200', - feature: 'processor', - module: 'destination', - version: 'v0', - input: { - request: { - body: [ - { - destination, - message: { - context, - type: 'page', - event: 'product purchased', - userId: 'sajal12', - channel: 'mobile', - messageId: 'dummy_msg_id', - properties: commonProperties, - anonymousId: 'anon_123', - integrations: { - All: true, - }, - originalTimestamp: '2021-01-25T15:32:56.409Z', - }, - metadata, - }, - ], - }, - }, - output: { - response: { - status: 200, - body: [ - { - metadata: { - destinationId: 'dummyDestId', - }, - output: transformResultBuilder({ - method: 'POST', - endpoint: - 'https://experience.ninetailed.co/v2/organizations/dummyOrganisationId/environments/main/events', - JSON: { - events: [ - { - context: { - app: { - name: 'RudderLabs JavaScript SDK', - version: '1.0.0', - }, - campaign: { - name: 'campign_123', - source: 'social marketing', - medium: 'facebook', - term: '1 year', - }, - library: { - name: 'RudderstackSDK', - version: 'Ruddderstack SDK version', - }, - locale: 'en-US', - page: { - path: '/signup', - referrer: 'https://rudderstack.medium.com/', - search: '?type=freetrial', - url: 'https://app.rudderstack.com/signup?type=freetrial', - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - location: { - coordinates: { - latitude: 40.7128, - longitude: -74.006, - }, - city: 'San Francisco', - postalCode: '94107', - region: 'CA', - regionCode: 'CA', - country: ' United States', - countryCode: 'United States of America', - continent: 'North America', - timezone: 'America/Los_Angeles', - }, - }, - type: 'page', - channel: 'mobile', - messageId: 'dummy_msg_id', - properties: commonProperties, - anonymousId: 'anon_123', - originalTimestamp: '2021-01-25T15:32:56.409Z', - }, - ], - }, - userId: '', - }), - statusCode: 200, - }, - ], - }, - }, - }, -]; diff --git a/test/integrations/destinations/ninetailed/router/data.ts b/test/integrations/destinations/ninetailed/router/data.ts index 05105f4aed..1bf664d1c4 100644 --- a/test/integrations/destinations/ninetailed/router/data.ts +++ b/test/integrations/destinations/ninetailed/router/data.ts @@ -31,15 +31,6 @@ export const data = [ metadata: { jobId: 1, userId: 'u1' }, destination, }, - { - message: { - ...commonInput, - type: 'page', - properties: pageProperties, - }, - metadata: { jobId: 2, userId: 'u1' }, - destination, - }, { message: { type: 'identify', @@ -80,11 +71,6 @@ export const data = [ event: 'product list viewed', properties: trackProperties, }, - { - ...commonOutput, - type: 'page', - properties: pageProperties, - }, { type: 'identify', ...commonOutput, @@ -103,7 +89,6 @@ export const data = [ }, metadata: [ { jobId: 1, userId: 'u1' }, - { jobId: 2, userId: 'u1' }, { jobId: 3, userId: 'u1' }, ], batched: true, @@ -142,21 +127,9 @@ export const data = [ { message: { ...commonInput, - type: 'page', - properties: { - title: 'Sample Page', - url: 'https://example.com/?utm_campaign=example_campaign&utm_content=example_content', - path: '/', - hash: '', - search: '?utm_campaign=example_campaign&utm_content=example_content', - width: '1920', - height: '1080', - query: { - utm_campaign: 'example_campaign', - utm_content: 'example_content', - }, - referrer: '', - }, + type: 'track', + event: 'product added', + properties: trackProperties, }, metadata: { jobId: 2, userId: 'u1' }, destination, @@ -210,8 +183,9 @@ export const data = [ }, { ...commonOutput, - type: 'page', - properties: pageProperties, + type: 'track', + event: 'product added', + properties: trackProperties, }, ], }, @@ -264,8 +238,9 @@ export const data = [ { message: { ...commonInput, - type: 'page', - properties: pageProperties, + type: 'track', + event: 'product added', + properties: trackProperties, }, metadata: { jobId: 2, userId: 'u1' }, destination, @@ -330,8 +305,9 @@ export const data = [ }, { ...commonOutput, - type: 'page', - properties: pageProperties, + type: 'track', + event: 'product added', + properties: trackProperties, }, ], }, From 667095fa8316cd95a066f15b848ad503c6b4af80 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:48:50 +0530 Subject: [PATCH 2/4] feat: update movable ink batch size (#3223) * feat: update movable ink batch size * test: add testcase for batching on max request size --- src/cdk/v2/destinations/movable_ink/config.js | 3 +- .../movable_ink/procWorkflow.yaml | 1 + .../destinations/movable_ink/rtWorkflow.yaml | 2 +- .../destinations/movable_ink/common.ts | 180 ++++++++++++++++ .../destinations/movable_ink/mocks.ts | 6 + .../movable_ink/processor/identify.ts | 2 +- .../movable_ink/processor/track.ts | 6 + .../movable_ink/processor/validation.ts | 44 ++++ .../destinations/movable_ink/router/data.ts | 199 +++++++++++++++++- 9 files changed, 437 insertions(+), 6 deletions(-) create mode 100644 test/integrations/destinations/movable_ink/mocks.ts diff --git a/src/cdk/v2/destinations/movable_ink/config.js b/src/cdk/v2/destinations/movable_ink/config.js index 673e94620e..9a0200ab44 100644 --- a/src/cdk/v2/destinations/movable_ink/config.js +++ b/src/cdk/v2/destinations/movable_ink/config.js @@ -1,3 +1,4 @@ module.exports = { - MAX_REQUEST_SIZE_IN_BYTES: 13500, + MAX_REQUEST_SIZE_IN_BYTES: 1000000, + MAX_BATCH_SIZE: 1000, }; diff --git a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml index 43dbb3cbce..394190049b 100644 --- a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml +++ b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml @@ -24,6 +24,7 @@ steps: $.assertConfig(.destination.Config.accessKey, "Access key is not present . Aborting"); $.assertConfig(.destination.Config.accessSecret, "Access Secret is not present. Aborting"); $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting"); + $.assert(!(messageType === {{$.EventType.TRACK}} && !(.message.event)), "Event name is not present. Aborting"); const userId = .message.().( {{{{$.getGenericPaths("userIdOnly")}}}}; diff --git a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml index 46afb34d53..3ffa49f15b 100644 --- a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml +++ b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml @@ -42,7 +42,7 @@ steps: description: Batches the successfulEvents template: | let batches = $.BatchUtils.chunkArrayBySizeAndLength( - $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES}).items; + $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES, maxItems: $.MAX_BATCH_SIZE}).items; batches@batch.({ "batchedRequest": { diff --git a/test/integrations/destinations/movable_ink/common.ts b/test/integrations/destinations/movable_ink/common.ts index f7eaa7af39..29fe76852c 100644 --- a/test/integrations/destinations/movable_ink/common.ts +++ b/test/integrations/destinations/movable_ink/common.ts @@ -110,6 +110,186 @@ const trackTestProperties = { position: 2, category: 'Games', }, + { + product_id: '122c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Ticket to Ride', + price: 20, + position: 3, + category: 'Games', + }, + { + product_id: '222c6f5d5cf86a4c77358033', + sku: '9472-998-0112', + name: 'Catan', + price: 30, + position: 4, + category: 'Games', + }, + { + product_id: '322c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Pandemic', + price: 25, + position: 5, + category: 'Games', + }, + { + product_id: '422c6f5d5cf86a4c77358033', + sku: '8472-998-0113', + name: 'Exploding Kittens', + price: 15, + position: 6, + category: 'Games', + }, + { + product_id: '522c6f5d5cf86a4c77358033', + sku: '8472-998-0114', + name: 'Codenames', + price: 18, + position: 7, + category: 'Games', + }, + { + product_id: '622c6f5d5cf86a4c77358034', + sku: '8472-998-0115', + name: 'Scythe', + price: 35, + position: 8, + category: 'Games', + }, + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + name: 'Cones of Dunshire', + price: 40, + position: 1, + category: 'Games', + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.jpg', + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + name: 'Five Crowns', + price: 5, + position: 2, + category: 'Games', + }, + { + product_id: '122c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Ticket to Ride', + price: 20, + position: 3, + category: 'Games', + }, + { + product_id: '222c6f5d5cf86a4c77358033', + sku: '9472-998-0112', + name: 'Catan', + price: 30, + position: 4, + category: 'Games', + }, + { + product_id: '322c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Pandemic', + price: 25, + position: 5, + category: 'Games', + }, + { + product_id: '422c6f5d5cf86a4c77358033', + sku: '8472-998-0113', + name: 'Exploding Kittens', + price: 15, + position: 6, + category: 'Games', + }, + { + product_id: '522c6f5d5cf86a4c77358033', + sku: '8472-998-0114', + name: 'Codenames', + price: 18, + position: 7, + category: 'Games', + }, + { + product_id: '622c6f5d5cf86a4c77358034', + sku: '8472-998-0115', + name: 'Scythe', + price: 35, + position: 8, + category: 'Games', + }, + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + name: 'Cones of Dunshire', + price: 40, + position: 1, + category: 'Games', + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.jpg', + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + name: 'Five Crowns', + price: 5, + position: 2, + category: 'Games', + }, + { + product_id: '122c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Ticket to Ride', + price: 20, + position: 3, + category: 'Games', + }, + { + product_id: '222c6f5d5cf86a4c77358033', + sku: '9472-998-0112', + name: 'Catan', + price: 30, + position: 4, + category: 'Games', + }, + { + product_id: '322c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Pandemic', + price: 25, + position: 5, + category: 'Games', + }, + { + product_id: '422c6f5d5cf86a4c77358033', + sku: '8472-998-0113', + name: 'Exploding Kittens', + price: 15, + position: 6, + category: 'Games', + }, + { + product_id: '522c6f5d5cf86a4c77358033', + sku: '8472-998-0114', + name: 'Codenames', + price: 18, + position: 7, + category: 'Games', + }, + { + product_id: '622c6f5d5cf86a4c77358034', + sku: '8472-998-0115', + name: 'Scythe', + price: 35, + position: 8, + category: 'Games', + }, ], }, 'Products Searched': { query: 'HDMI cable', url: 'https://www.website.com/product/path' }, diff --git a/test/integrations/destinations/movable_ink/mocks.ts b/test/integrations/destinations/movable_ink/mocks.ts new file mode 100644 index 0000000000..2468f51315 --- /dev/null +++ b/test/integrations/destinations/movable_ink/mocks.ts @@ -0,0 +1,6 @@ +import config from '../../../../src/cdk/v2/destinations/movable_ink/config'; + +export const defaultMockFns = () => { + jest.replaceProperty(config, 'MAX_REQUEST_SIZE_IN_BYTES', 5000); + jest.replaceProperty(config, 'MAX_BATCH_SIZE', 2); +}; diff --git a/test/integrations/destinations/movable_ink/processor/identify.ts b/test/integrations/destinations/movable_ink/processor/identify.ts index 27186da05c..e5bbf5a9a7 100644 --- a/test/integrations/destinations/movable_ink/processor/identify.ts +++ b/test/integrations/destinations/movable_ink/processor/identify.ts @@ -1,6 +1,6 @@ import { ProcessorTestData } from '../../../testTypes'; import { generateMetadata, transformResultBuilder } from '../../../testUtils'; -import { destType, channel, destination, traits, headers } from '../common'; +import { destType, destination, traits, headers } from '../common'; export const identify: ProcessorTestData[] = [ { diff --git a/test/integrations/destinations/movable_ink/processor/track.ts b/test/integrations/destinations/movable_ink/processor/track.ts index 5f30a3de83..890de11a0c 100644 --- a/test/integrations/destinations/movable_ink/processor/track.ts +++ b/test/integrations/destinations/movable_ink/processor/track.ts @@ -23,6 +23,7 @@ export const track: ProcessorTestData[] = [ channel, anonymousId: 'anonId123', userId: 'userId123', + event: 'Product Added', properties: trackTestProperties['Product Added'], integrations: { All: true, @@ -49,6 +50,7 @@ export const track: ProcessorTestData[] = [ channel, userId: 'userId123', anonymousId: 'anonId123', + event: 'Product Added', properties: trackTestProperties['Product Added'], integrations: { All: true, @@ -84,6 +86,7 @@ export const track: ProcessorTestData[] = [ channel, anonymousId: 'anonId123', userId: 'userId123', + event: 'Order Completed', properties: trackTestProperties['Order Completed'], integrations: { All: true, @@ -110,6 +113,7 @@ export const track: ProcessorTestData[] = [ channel, userId: 'userId123', anonymousId: 'anonId123', + event: 'Order Completed', properties: trackTestProperties['Order Completed'], integrations: { All: true, @@ -145,6 +149,7 @@ export const track: ProcessorTestData[] = [ channel, anonymousId: 'anonId123', userId: 'userId123', + event: 'Custom Event', properties: trackTestProperties['Custom Event'], integrations: { All: true, @@ -171,6 +176,7 @@ export const track: ProcessorTestData[] = [ channel, userId: 'userId123', anonymousId: 'anonId123', + event: 'Custom Event', properties: trackTestProperties['Custom Event'], integrations: { All: true, diff --git a/test/integrations/destinations/movable_ink/processor/validation.ts b/test/integrations/destinations/movable_ink/processor/validation.ts index ab6b123eb7..6aafb5e2c0 100644 --- a/test/integrations/destinations/movable_ink/processor/validation.ts +++ b/test/integrations/destinations/movable_ink/processor/validation.ts @@ -214,4 +214,48 @@ export const validation: ProcessorTestData[] = [ }, }, }, + { + id: 'MovableInk-validation-test-6', + name: destType, + description: 'Missing event name', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + anonymousId: 'anonId123', + userId: 'userId123', + properties: {}, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Event name is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Event name is not present. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/movable_ink/router/data.ts b/test/integrations/destinations/movable_ink/router/data.ts index 72df3d7074..afadfec56e 100644 --- a/test/integrations/destinations/movable_ink/router/data.ts +++ b/test/integrations/destinations/movable_ink/router/data.ts @@ -1,6 +1,7 @@ import { RouterTestData } from '../../../testTypes'; import { RouterTransformationRequest } from '../../../../../src/types'; import { generateMetadata } from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; import { destType, channel, @@ -43,6 +44,7 @@ const routerRequest: RouterTransformationRequest = { channel, anonymousId: 'anonId123', userId: 'userId123', + event: 'Product Added', properties: trackTestProperties['Product Added'], integrations: { All: true, @@ -58,19 +60,73 @@ const routerRequest: RouterTransformationRequest = { channel, anonymousId: 'anonId123', userId: 'userId123', - properties: trackTestProperties['Custom Event'], + event: 'Custom Event', integrations: { All: true, }, + originalTimestamp: '2024-03-04T15:32:56.409Z', }, metadata: generateMetadata(4), destination, }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + event: 'Custom Event', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + }, + metadata: generateMetadata(5), + destination, + }, + ], + destType, +}; + +// >5KB payload +const routerRequest2: RouterTransformationRequest = { + input: [ + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + event: 'Order Completed', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + event: 'Custom Event', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(2), + destination, + }, ], destType, }; -export const data: RouterTestData[] = [ +export const data = [ { id: 'MovableInk-router-test-1', name: destType, @@ -118,6 +174,7 @@ export const data: RouterTestData[] = [ channel, userId: 'userId123', anonymousId: 'anonId123', + event: 'Product Added', properties: trackTestProperties['Product Added'], integrations: { All: true, @@ -138,6 +195,42 @@ export const data: RouterTestData[] = [ statusCode: 200, destination, }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + events: [ + { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + event: 'Custom Event', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(4)], + batched: true, + statusCode: 200, + destination, + }, { metadata: [generateMetadata(2)], batched: false, @@ -147,7 +240,7 @@ export const data: RouterTestData[] = [ destination, }, { - metadata: [generateMetadata(4)], + metadata: [generateMetadata(5)], batched: false, statusCode: 400, error: 'Timestamp is not present. Aborting', @@ -158,5 +251,105 @@ export const data: RouterTestData[] = [ }, }, }, + mockFns: defaultMockFns, + }, + { + id: 'MovableInk-router-test-2', + name: destType, + description: 'Basic Router Test to test Max Request Size', + scenario: 'Framework', + successCriteria: + 'Some events should be transformed successfully and some should fail for missing fields and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest2, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + events: [ + { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + event: 'Order Completed', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + events: [ + { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + event: 'Custom Event', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(2)], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, }, ]; From d9b7e1f70565d59979aee3e62f60e39edb9a23c7 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Mon, 1 Apr 2024 15:10:10 +0530 Subject: [PATCH 3/4] feat: onboard new destination bloomreach (#3185) * feat: onboard new destination bloomreach * refactor: move code to typescript * feat: add proxy and partial handling support * feat: onboard on router * chore: add proxy testcases * docs: add ref --- src/cdk/v2/destinations/bloomreach/config.ts | 30 +++ .../BloomreachCustomerPropertiesConfig.json | 36 +++ .../destinations/bloomreach/procWorkflow.yaml | 119 ++++++++++ .../destinations/bloomreach/rtWorkflow.yaml | 76 ++++++ src/cdk/v2/destinations/bloomreach/utils.ts | 31 +++ src/constants/destinationCanonicalNames.js | 1 + src/features.json | 5 +- .../destinations/bloomreach/networkHandler.js | 83 +++++++ .../destinations/bloomreach/common.ts | 99 ++++++++ .../bloomreach/dataDelivery/business.ts | 195 ++++++++++++++++ .../bloomreach/dataDelivery/data.ts | 3 + .../bloomreach/dataDelivery/other.ts | 212 +++++++++++++++++ .../destinations/bloomreach/mocks.ts | 5 + .../destinations/bloomreach/network.ts | 124 ++++++++++ .../destinations/bloomreach/processor/data.ts | 5 + .../bloomreach/processor/identify.ts | 156 +++++++++++++ .../destinations/bloomreach/processor/page.ts | 72 ++++++ .../bloomreach/processor/track.ts | 173 ++++++++++++++ .../bloomreach/processor/validation.ts | 131 +++++++++++ .../destinations/bloomreach/router/data.ts | 220 ++++++++++++++++++ 20 files changed, 1774 insertions(+), 2 deletions(-) create mode 100644 src/cdk/v2/destinations/bloomreach/config.ts create mode 100644 src/cdk/v2/destinations/bloomreach/data/BloomreachCustomerPropertiesConfig.json create mode 100644 src/cdk/v2/destinations/bloomreach/procWorkflow.yaml create mode 100644 src/cdk/v2/destinations/bloomreach/rtWorkflow.yaml create mode 100644 src/cdk/v2/destinations/bloomreach/utils.ts create mode 100644 src/v1/destinations/bloomreach/networkHandler.js create mode 100644 test/integrations/destinations/bloomreach/common.ts create mode 100644 test/integrations/destinations/bloomreach/dataDelivery/business.ts create mode 100644 test/integrations/destinations/bloomreach/dataDelivery/data.ts create mode 100644 test/integrations/destinations/bloomreach/dataDelivery/other.ts create mode 100644 test/integrations/destinations/bloomreach/mocks.ts create mode 100644 test/integrations/destinations/bloomreach/network.ts create mode 100644 test/integrations/destinations/bloomreach/processor/data.ts create mode 100644 test/integrations/destinations/bloomreach/processor/identify.ts create mode 100644 test/integrations/destinations/bloomreach/processor/page.ts create mode 100644 test/integrations/destinations/bloomreach/processor/track.ts create mode 100644 test/integrations/destinations/bloomreach/processor/validation.ts create mode 100644 test/integrations/destinations/bloomreach/router/data.ts diff --git a/src/cdk/v2/destinations/bloomreach/config.ts b/src/cdk/v2/destinations/bloomreach/config.ts new file mode 100644 index 0000000000..90fbcc63c6 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach/config.ts @@ -0,0 +1,30 @@ +import { getMappingConfig } from '../../../../v0/util'; + +export const CUSTOMER_COMMAND = 'customers'; +export const CUSTOMER_EVENT_COMMAND = 'customers/events'; +export const MAX_BATCH_SIZE = 50; + +// ref:- https://documentation.bloomreach.com/engagement/reference/batch-commands-2 +export const getBatchEndpoint = (apiBaseUrl: string, projectToken: string): string => + `${apiBaseUrl}/track/v2/projects/${projectToken}/batch`; + +const CONFIG_CATEGORIES = { + CUSTOMER_PROPERTIES_CONFIG: { name: 'BloomreachCustomerPropertiesConfig' }, +}; +const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); +export const EXCLUSION_FIELDS: string[] = [ + 'email', + 'firstName', + 'firstname', + 'first_name', + 'lastName', + 'lastname', + 'last_name', + 'name', + 'phone', + 'city', + 'birthday', + 'country', +]; +export const CUSTOMER_PROPERTIES_CONFIG = + MAPPING_CONFIG[CONFIG_CATEGORIES.CUSTOMER_PROPERTIES_CONFIG.name]; diff --git a/src/cdk/v2/destinations/bloomreach/data/BloomreachCustomerPropertiesConfig.json b/src/cdk/v2/destinations/bloomreach/data/BloomreachCustomerPropertiesConfig.json new file mode 100644 index 0000000000..cb4c2f7201 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach/data/BloomreachCustomerPropertiesConfig.json @@ -0,0 +1,36 @@ +[ + { + "destKey": "first_name", + "sourceKeys": "firstName", + "sourceFromGenericMap": true + }, + { + "destKey": "last_name", + "sourceKeys": "lastName", + "sourceFromGenericMap": true + }, + { + "destKey": "email", + "sourceKeys": "emailOnly", + "sourceFromGenericMap": true + }, + { + "destKey": "phone", + "sourceKeys": "phone", + "sourceFromGenericMap": true + }, + { + "destKey": "city", + "sourceKeys": "city", + "sourceFromGenericMap": true + }, + { + "destKey": "country", + "sourceKeys": ["traits.address.country", "context.traits.address.country"] + }, + { + "destKey": "birthday", + "sourceKeys": "birthday", + "sourceFromGenericMap": true + } +] diff --git a/src/cdk/v2/destinations/bloomreach/procWorkflow.yaml b/src/cdk/v2/destinations/bloomreach/procWorkflow.yaml new file mode 100644 index 0000000000..f092d90382 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach/procWorkflow.yaml @@ -0,0 +1,119 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: defaultRequestConfig + path: ../../../../v0/util + - name: toUnixTimestamp + path: ../../../../v0/util + - name: base64Convertor + path: ../../../../v0/util + - name: removeUndefinedAndNullValues + path: ../../../../v0/util + - name: generateExclusionList + path: ../../../../v0/util + - name: extractCustomFields + path: ../../../../v0/util + - name: constructPayload + path: ../../../../v0/util + - path: ./utils + - path: ./config + +steps: + - name: messageType + template: | + $.context.messageType = .message.type.toLowerCase(); + + - name: validateInput + template: | + let messageType = $.context.messageType; + $.assert(messageType, "message Type is not present. Aborting"); + $.assert(messageType in {{$.EventType.([.IDENTIFY,.TRACK,.PAGE,.SCREEN])}}, "message type " + messageType + " is not supported"); + $.assertConfig(.destination.Config.apiBaseUrl, "API Base URL is not present. Aborting"); + $.assertConfig(.destination.Config.apiKey, "API Key is not present . Aborting"); + $.assertConfig(.destination.Config.apiSecret, "API Secret is not present. Aborting"); + $.assertConfig(.destination.Config.projectToken, "Project Token is not present. Aborting"); + $.assertConfig(.destination.Config.hardID, "Hard ID is not present. Aborting"); + $.assertConfig(.destination.Config.softID, "Soft ID is not present. Aborting"); + $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting"); + const userId = .message.().( + {{{{$.getGenericPaths("userIdOnly")}}}}; + ); + $.assert(userId ?? .message.anonymousId, "Either one of userId or anonymousId is required. Aborting"); + + - name: prepareIdentifyPayload + condition: $.context.messageType === {{$.EventType.IDENTIFY}} + template: | + const customerIDs = $.prepareCustomerIDs(.message, .destination); + const customerProperties = $.constructPayload(.message, $.CUSTOMER_PROPERTIES_CONFIG); + const extraCustomerProperties = $.extractCustomFields(.message, {}, ['traits', 'context.traits'], $.EXCLUSION_FIELDS); + const properties = { + ...customerProperties, + ...extraCustomerProperties + } + const data = .message.().({ + "customer_ids": customerIDs, + "update_timestamp": $.toUnixTimestamp({{{{$.getGenericPaths("timestamp")}}}}), + properties + }); + + $.context.payload = $.removeUndefinedAndNullValues({name: $.CUSTOMER_COMMAND, data}) + + - name: prepareEventName + steps: + - name: pageEventName + condition: $.context.messageType === {{$.EventType.PAGE}} + template: | + const category = .message.category ?? .message.properties.category; + const name = .message.name || .message.properties.name; + const eventNameArray = ["Viewed"]; + category ? eventNameArray.push(category); + name ? eventNameArray.push(name); + eventNameArray.push("Page"); + $.context.event = eventNameArray.join(" "); + - name: screenEventName + condition: $.context.messageType === {{$.EventType.SCREEN}} + template: | + const category = .message.category ?? .message.properties.category; + const name = .message.name || .message.properties.name; + const eventNameArray = ["Viewed"]; + category ? eventNameArray.push(category); + name ? eventNameArray.push(name); + eventNameArray.push("Screen"); + $.context.event = eventNameArray.join(" "); + - name: trackEventName + condition: $.context.messageType === {{$.EventType.TRACK}} + template: | + $.assert(.message.event, "Event name is required. Aborting"); + $.context.event = .message.event + + - name: prepareTrackPageScreenPayload + condition: $.context.messageType !== {{$.EventType.IDENTIFY}} + template: | + const customerIDs = $.prepareCustomerIDs(.message, .destination); + const data = .message.().({ + "customer_ids": customerIDs, + "timestamp": $.toUnixTimestamp({{{{$.getGenericPaths("timestamp")}}}}), + "properties": .properties, + "event_type": $.context.event, + }); + + $.context.payload = $.removeUndefinedAndNullValues({name: $.CUSTOMER_EVENT_COMMAND, data}) + + - name: buildResponse + description: In batchMode we return payload directly + condition: $.batchMode + template: | + $.context.payload + else: + name: buildResponseForProcessTransformation + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = $.context.payload; + response.endpoint = $.getBatchEndpoint(.destination.Config.apiBaseUrl, .destination.Config.projectToken); + response.method = "POST"; + response.headers = { + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.destination.Config.apiKey + ":" + .destination.Config.apiSecret) + } + response; diff --git a/src/cdk/v2/destinations/bloomreach/rtWorkflow.yaml b/src/cdk/v2/destinations/bloomreach/rtWorkflow.yaml new file mode 100644 index 0000000000..b8b27ca02e --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach/rtWorkflow.yaml @@ -0,0 +1,76 @@ +bindings: + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - path: ./utils + exportAll: true + - name: base64Convertor + path: ../../../../v0/util + - name: toUnixTimestamp + path: ../../../../v0/util + - name: BatchUtils + path: '@rudderstack/workflow-engine' + - path: ./config + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + bindings: + - name: batchMode + value: true + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "batchedRequest": ., + "batched": false, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata, + "statusCode": 200 + })[] + + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + let batches = $.BatchUtils.chunkArrayBySizeAndLength( + $.outputs.successfulEvents, {maxItems: $.MAX_BATCH_SIZE}).items; + + batches@batch.({ + "batchedRequest": { + "body": { + "JSON": {"commands": ~r batch.batchedRequest[]}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": batch[0].destination.Config.().($.getBatchEndpoint(.apiBaseUrl, .projectToken)), + "headers": batch[0].destination.Config.().({ + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.apiKey + ":" + .apiSecret) + }), + "params": {}, + "files": {} + }, + "metadata": ~r batch.metadata[], + "batched": true, + "statusCode": 200, + "destination": batch[0].destination + })[]; + + - name: finalPayload + template: | + [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents] diff --git a/src/cdk/v2/destinations/bloomreach/utils.ts b/src/cdk/v2/destinations/bloomreach/utils.ts new file mode 100644 index 0000000000..f834fa74e7 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach/utils.ts @@ -0,0 +1,31 @@ +import { isObject, isEmptyObject, getIntegrationsObj } from '../../../../v0/util'; +import { RudderMessage, Destination } from '../../../../types'; + +const getCustomerIDsFromIntegrationObject = (message: RudderMessage): any => { + const integrationObj = getIntegrationsObj(message, 'bloomreach' as any) || {}; + const { hardID, softID } = integrationObj; + const customerIDs = {}; + + if (isObject(hardID) && !isEmptyObject(hardID)) { + Object.keys(hardID).forEach((id) => { + customerIDs[id] = hardID[id]; + }); + } + + if (isObject(softID) && !isEmptyObject(softID)) { + Object.keys(softID).forEach((id) => { + customerIDs[id] = softID[id]; + }); + } + + return customerIDs; +}; + +export const prepareCustomerIDs = (message: RudderMessage, destination: Destination): any => { + const customerIDs = { + [destination.Config.hardID]: message.userId, + [destination.Config.softID]: message.anonymousId, + ...getCustomerIDsFromIntegrationObject(message), + }; + return customerIDs; +}; diff --git a/src/constants/destinationCanonicalNames.js b/src/constants/destinationCanonicalNames.js index b84aff1089..ee4f4f0b33 100644 --- a/src/constants/destinationCanonicalNames.js +++ b/src/constants/destinationCanonicalNames.js @@ -165,6 +165,7 @@ const DestCanonicalNames = { 'google adwords offline conversions', ], koala: ['Koala', 'koala', 'KOALA'], + bloomreach: ['Bloomreach', 'bloomreach', 'BLOOMREACH'], }; module.exports = { DestHandlerMap, DestCanonicalNames }; diff --git a/src/features.json b/src/features.json index 267923fdb4..fd06b3b241 100644 --- a/src/features.json +++ b/src/features.json @@ -67,8 +67,9 @@ "THE_TRADE_DESK": true, "INTERCOM": true, "NINETAILED": true, - "MOVABLE_INK": true, - "KOALA": true + "KOALA": true, + "BLOOMREACH": true, + "MOVABLE_INK": true }, "regulations": [ "BRAZE", diff --git a/src/v1/destinations/bloomreach/networkHandler.js b/src/v1/destinations/bloomreach/networkHandler.js new file mode 100644 index 0000000000..a3c17a167b --- /dev/null +++ b/src/v1/destinations/bloomreach/networkHandler.js @@ -0,0 +1,83 @@ +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); +const { proxyRequest, prepareProxyRequest } = require('../../../adapters/network'); +const { + processAxiosResponse, + getDynamicErrorType, +} = require('../../../adapters/utils/networkUtils'); +const { isHttpStatusSuccess } = require('../../../v0/util/index'); +const tags = require('../../../v0/util/tags'); + +// { +// "results": [ +// { +// "success": true +// }, +// { +// "success": false, +// "errors": [ +// "At least one id should be specified." +// ] +// } +// ], +// "start_time": 1710750816.8504393, +// "end_time": 1710750816.8518236, +// "success": true +// } +const checkIfEventIsAbortableAndExtractErrorMessage = (element) => { + if (element.success) { + return { isAbortable: false, errorMsg: '' }; + } + + const errorMsg = element.errors.join(', '); + return { isAbortable: true, errorMsg }; +}; + +const responseHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata } = responseParams; + + const message = '[BLOOMREACH Response V1 Handler] - Request Processed Successfully'; + const responseWithIndividualEvents = []; + const { response, status } = destinationResponse; + + if (isHttpStatusSuccess(status)) { + // check for Partial Event failures and Successes + const { results } = response; + results.forEach((event, idx) => { + const proxyOutput = { + statusCode: 200, + metadata: rudderJobMetadata[idx], + error: 'success', + }; + // update status of partial event if abortable + const { isAbortable, errorMsg } = checkIfEventIsAbortableAndExtractErrorMessage(event); + if (isAbortable) { + proxyOutput.statusCode = 400; + proxyOutput.error = errorMsg; + } + responseWithIndividualEvents.push(proxyOutput); + }); + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; + } + throw new TransformerProxyError( + `BLOOMREACH: Error encountered in transformer proxy V1`, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + '', + responseWithIndividualEvents, + ); +}; +function networkHandler() { + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.prepareProxy = prepareProxyRequest; + this.responseHandler = responseHandler; +} +module.exports = { networkHandler }; diff --git a/test/integrations/destinations/bloomreach/common.ts b/test/integrations/destinations/bloomreach/common.ts new file mode 100644 index 0000000000..798e744cbc --- /dev/null +++ b/test/integrations/destinations/bloomreach/common.ts @@ -0,0 +1,99 @@ +import { Destination } from '../../../../src/types'; + +const destType = 'bloomreach'; +const destTypeInUpperCase = 'BLOOMREACH'; +const displayName = 'bloomreach'; +const channel = 'web'; +const destination: Destination = { + Config: { + apiBaseUrl: 'https://demoapp-api.bloomreach.com', + apiKey: 'test-api-key', + apiSecret: 'test-api-secret', + projectToken: 'test-project-token', + hardID: 'registered', + softID: 'cookie', + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', +}; + +const traits = { + email: 'test@example.com', + firstName: 'John', + lastName: 'Doe', + phone: '1234567890', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, +}; + +const properties = { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + category: 'Games', + name: 'Cones of Dunshire', + brand: 'Wyatt Games', + variant: 'expansion pack', + price: 49.99, + quantity: 5, + coupon: 'PREORDER15', + currency: 'USD', + position: 1, + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.webp', + key1: 'value1', +}; +const endpoint = 'https://demoapp-api.bloomreach.com/track/v2/projects/test-project-token/batch'; + +const processorInstrumentationErrorStatTags = { + destType: destTypeInUpperCase, + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', +}; + +const RouterInstrumentationErrorStatTags = { + ...processorInstrumentationErrorStatTags, + feature: 'router', +}; + +const proxyV1RetryableErrorStatTags = { + ...RouterInstrumentationErrorStatTags, + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', +}; + +const headers = { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC1hcGkta2V5OnRlc3QtYXBpLXNlY3JldA==', +}; + +export { + destType, + channel, + destination, + processorInstrumentationErrorStatTags, + RouterInstrumentationErrorStatTags, + traits, + headers, + properties, + endpoint, + proxyV1RetryableErrorStatTags, +}; diff --git a/test/integrations/destinations/bloomreach/dataDelivery/business.ts b/test/integrations/destinations/bloomreach/dataDelivery/business.ts new file mode 100644 index 0000000000..9e71b7a2fd --- /dev/null +++ b/test/integrations/destinations/bloomreach/dataDelivery/business.ts @@ -0,0 +1,195 @@ +import { ProxyV1TestData } from '../../../testTypes'; +import { generateProxyV1Payload, generateMetadata } from '../../../testUtils'; +import { destType, headers, properties, endpoint } from '../common'; + +const customerProperties = { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, +}; + +const metadataArray = [generateMetadata(1), generateMetadata(2)]; + +// https://documentation.bloomreach.com/engagement/reference/tips-and-best-practices +export const businessProxyV1: ProxyV1TestData[] = [ + { + id: 'bloomreach_v1_business_scenario_1', + name: destType, + description: + '[Proxy v1 API] :: Test for a valid request - where the destination responds with 200 with error for request 2 in a batch', + successCriteria: 'Should return 200 with partial failures within the response payload', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers, + params: {}, + JSON: { + commands: [ + { + name: 'customers', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + update_timestamp: 1709405952, + properties: customerProperties, + }, + }, + { + name: 'customers', + data: { + customer_ids: {}, + }, + }, + ], + }, + endpoint, + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[BLOOMREACH Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + results: [ + { + success: true, + }, + { + success: false, + errors: ['At least one id should be specified.'], + }, + ], + start_time: 1710771351.9885373, + end_time: 1710771351.9891083, + success: true, + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + { + statusCode: 400, + metadata: generateMetadata(2), + error: 'At least one id should be specified.', + }, + ], + }, + }, + }, + }, + }, + { + id: 'bloomreach_v1_business_scenario_2', + name: destType, + description: + '[Proxy v1 API] :: Test for a valid request - where the destination responds with 200 without any error', + successCriteria: 'Should return 200 with no error with destination response', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers, + params: {}, + JSON: { + commands: [ + { + name: 'customers/events', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + timestamp: 1709566376, + properties, + event_type: 'test_event', + }, + }, + { + name: 'customers', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + update_timestamp: 1709405952, + properties: customerProperties, + }, + }, + ], + }, + endpoint, + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[BLOOMREACH Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + results: [ + { + success: true, + }, + { + success: true, + }, + ], + start_time: 1710771351.9885373, + end_time: 1710771351.9891083, + success: true, + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + { + statusCode: 200, + metadata: generateMetadata(2), + error: 'success', + }, + ], + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/dataDelivery/data.ts b/test/integrations/destinations/bloomreach/dataDelivery/data.ts new file mode 100644 index 0000000000..5099eafce7 --- /dev/null +++ b/test/integrations/destinations/bloomreach/dataDelivery/data.ts @@ -0,0 +1,3 @@ +import { businessProxyV1 } from './business'; +import { otherProxyV1 } from './other'; +export const data = [...businessProxyV1, ...otherProxyV1]; diff --git a/test/integrations/destinations/bloomreach/dataDelivery/other.ts b/test/integrations/destinations/bloomreach/dataDelivery/other.ts new file mode 100644 index 0000000000..f0dd9cc09a --- /dev/null +++ b/test/integrations/destinations/bloomreach/dataDelivery/other.ts @@ -0,0 +1,212 @@ +import { ProxyV1TestData } from '../../../testTypes'; +import { generateProxyV1Payload, generateMetadata } from '../../../testUtils'; +import { destType, proxyV1RetryableErrorStatTags } from '../common'; + +const metadataArray = [generateMetadata(1)]; + +// https://documentation.bloomreach.com/engagement/reference/tips-and-best-practices +export const otherProxyV1: ProxyV1TestData[] = [ + { + id: 'bloomreach_v1_other_scenario_1', + name: destType, + description: + '[Proxy v1 API] :: Scenario for testing Service Unavailable error from destination', + successCriteria: 'Should return 500 status code with error message', + scenario: 'Framework', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://random_test_url/test_for_service_not_available', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: + '{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}', + statusCode: 503, + metadata: generateMetadata(1), + }, + ], + statTags: proxyV1RetryableErrorStatTags, + message: 'BLOOMREACH: Error encountered in transformer proxy V1', + status: 503, + }, + }, + }, + }, + }, + { + id: 'bloomreach_v1_other_scenario_2', + name: destType, + description: '[Proxy v1 API] :: Scenario for testing Internal Server error from destination', + successCriteria: 'Should return 500 status code with error message', + scenario: 'Framework', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://random_test_url/test_for_internal_server_error', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: '"Internal Server Error"', + statusCode: 500, + metadata: generateMetadata(1), + }, + ], + statTags: proxyV1RetryableErrorStatTags, + message: 'BLOOMREACH: Error encountered in transformer proxy V1', + status: 500, + }, + }, + }, + }, + }, + { + id: 'bloomreach_v1_other_scenario_3', + name: destType, + description: '[Proxy v1 API] :: Scenario for testing Gateway Time Out error from destination', + successCriteria: 'Should return 504 status code with error message', + scenario: 'Framework', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://random_test_url/test_for_gateway_time_out', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: '"Gateway Timeout"', + statusCode: 504, + metadata: generateMetadata(1), + }, + ], + statTags: proxyV1RetryableErrorStatTags, + message: 'BLOOMREACH: Error encountered in transformer proxy V1', + status: 504, + }, + }, + }, + }, + }, + { + id: 'bloomreach_v1_other_scenario_4', + name: destType, + description: '[Proxy v1 API] :: Scenario for testing null response from destination', + successCriteria: 'Should return 500 status code with error message', + scenario: 'Framework', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://random_test_url/test_for_null_response', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: '""', + statusCode: 500, + metadata: generateMetadata(1), + }, + ], + statTags: proxyV1RetryableErrorStatTags, + message: 'BLOOMREACH: Error encountered in transformer proxy V1', + status: 500, + }, + }, + }, + }, + }, + { + id: 'bloomreach_v1_other_scenario_5', + name: destType, + description: + '[Proxy v1 API] :: Scenario for testing null and no status response from destination', + successCriteria: 'Should return 500 status code with error message', + scenario: 'Framework', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://random_test_url/test_for_null_and_no_status', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: '""', + statusCode: 500, + metadata: generateMetadata(1), + }, + ], + statTags: proxyV1RetryableErrorStatTags, + message: 'BLOOMREACH: Error encountered in transformer proxy V1', + status: 500, + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/mocks.ts b/test/integrations/destinations/bloomreach/mocks.ts new file mode 100644 index 0000000000..ba3b22b52a --- /dev/null +++ b/test/integrations/destinations/bloomreach/mocks.ts @@ -0,0 +1,5 @@ +import * as config from '../../../../src/cdk/v2/destinations/bloomreach/config'; + +export const defaultMockFns = () => { + jest.replaceProperty(config, 'MAX_BATCH_SIZE', 3 as typeof config.MAX_BATCH_SIZE); +}; diff --git a/test/integrations/destinations/bloomreach/network.ts b/test/integrations/destinations/bloomreach/network.ts new file mode 100644 index 0000000000..b20ff881b8 --- /dev/null +++ b/test/integrations/destinations/bloomreach/network.ts @@ -0,0 +1,124 @@ +import { destType, headers, properties, endpoint } from './common'; + +export const networkCallsData = [ + { + httpReq: { + url: endpoint, + data: { + commands: [ + { + name: 'customers', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + update_timestamp: 1709405952, + properties: { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, + }, + }, + }, + { + name: 'customers', + data: { + customer_ids: {}, + }, + }, + ], + }, + params: { destination: destType }, + headers, + method: 'POST', + }, + httpRes: { + data: { + results: [ + { + success: true, + }, + { + success: false, + errors: ['At least one id should be specified.'], + }, + ], + start_time: 1710771351.9885373, + end_time: 1710771351.9891083, + success: true, + }, + status: 200, + statusText: 'Ok', + }, + }, + { + httpReq: { + url: endpoint, + data: { + commands: [ + { + name: 'customers/events', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + timestamp: 1709566376, + properties, + event_type: 'test_event', + }, + }, + { + name: 'customers', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + update_timestamp: 1709405952, + properties: { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, + }, + }, + }, + ], + }, + params: { destination: destType }, + headers, + method: 'POST', + }, + httpRes: { + data: { + results: [ + { + success: true, + }, + { + success: true, + }, + ], + start_time: 1710771351.9885373, + end_time: 1710771351.9891083, + success: true, + }, + status: 200, + statusText: 'Ok', + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/processor/data.ts b/test/integrations/destinations/bloomreach/processor/data.ts new file mode 100644 index 0000000000..a3633ad0dd --- /dev/null +++ b/test/integrations/destinations/bloomreach/processor/data.ts @@ -0,0 +1,5 @@ +import { validation } from './validation'; +import { identify } from './identify'; +import { track } from './track'; +import { page } from './page'; +export const data = [...identify, ...track, ...page, ...validation]; diff --git a/test/integrations/destinations/bloomreach/processor/identify.ts b/test/integrations/destinations/bloomreach/processor/identify.ts new file mode 100644 index 0000000000..2a79cb57e3 --- /dev/null +++ b/test/integrations/destinations/bloomreach/processor/identify.ts @@ -0,0 +1,156 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, destination, traits, headers, endpoint } from '../common'; + +export const identify: ProcessorTestData[] = [ + { + id: 'bloomreach-identify-test-1', + name: destType, + description: 'Identify call to create/update customer properties', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + userId: 'userId123', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { registered: 'userId123', cookie: 'anonId123' }, + properties: { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, + }, + update_timestamp: 1709566376, + }, + name: 'customers', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'bloomreach-identify-test-2', + name: destType, + description: 'Identify call with multiple hard and soft identifiers using integration object', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain multiple hard and soft identifiers and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + userId: 'userId123', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + bloomreach: { + hardID: { + hardID1: 'value1', + }, + softID: { + google_analytics: 'gaId123', + softID2: 'value2', + }, + }, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { + registered: 'userId123', + cookie: 'anonId123', + hardID1: 'value1', + google_analytics: 'gaId123', + softID2: 'value2', + }, + properties: { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, + }, + update_timestamp: 1709566376, + }, + name: 'customers', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/processor/page.ts b/test/integrations/destinations/bloomreach/processor/page.ts new file mode 100644 index 0000000000..0c2d27989d --- /dev/null +++ b/test/integrations/destinations/bloomreach/processor/page.ts @@ -0,0 +1,72 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, destination, headers, endpoint } from '../common'; + +const properties = { + category: 'Docs', + path: '', + referrer: '', + search: '', + title: '', + url: '', +}; + +export const page: ProcessorTestData[] = [ + { + id: 'bloomreach-page-test-1', + name: destType, + description: 'Page call with category, name', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain event_name = "Viewed {{ category }} {{ name }} Page" and properties and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'page', + anonymousId: 'anonId123', + name: 'Integration', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { cookie: 'anonId123' }, + properties, + timestamp: 1709566376, + event_type: 'Viewed Docs Integration Page', + }, + name: 'customers/events', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/processor/track.ts b/test/integrations/destinations/bloomreach/processor/track.ts new file mode 100644 index 0000000000..a369f508b2 --- /dev/null +++ b/test/integrations/destinations/bloomreach/processor/track.ts @@ -0,0 +1,173 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, destination, headers, properties, endpoint } from '../common'; + +export const track: ProcessorTestData[] = [ + { + id: 'bloomreach-track-test-1', + name: destType, + description: 'Track call with anonymous user', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + anonymousId: 'anonId123', + event: 'Product Viewed', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { cookie: 'anonId123' }, + properties, + timestamp: 1709566376, + event_type: 'Product Viewed', + }, + name: 'customers/events', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'bloomreach-track-test-2', + name: destType, + description: 'Track call with known user', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + userId: 'userId123', + anonymousId: 'anonId123', + event: 'Product Added', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { registered: 'userId123', cookie: 'anonId123' }, + properties, + timestamp: 1709566376, + event_type: 'Product Added', + }, + name: 'customers/events', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'bloomreach-track-test-3', + name: destType, + description: 'Track call with no properties', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + anonymousId: 'anonId123', + event: 'test_event', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { cookie: 'anonId123' }, + timestamp: 1709566376, + event_type: 'test_event', + }, + name: 'customers/events', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/processor/validation.ts b/test/integrations/destinations/bloomreach/processor/validation.ts new file mode 100644 index 0000000000..ff959d74c6 --- /dev/null +++ b/test/integrations/destinations/bloomreach/processor/validation.ts @@ -0,0 +1,131 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata } from '../../../testUtils'; +import { destType, destination, processorInstrumentationErrorStatTags } from '../common'; + +export const validation: ProcessorTestData[] = [ + { + id: 'bloomreach-validation-test-1', + name: destType, + description: 'Missing userId and anonymousId', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Either one of userId or anonymousId is required. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Either one of userId or anonymousId is required. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'bloomreach-validation-test-2', + name: destType, + description: 'Unsupported message type -> group', + scenario: 'Framework', + successCriteria: 'Instrumentation Error for Unsupported message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'group', + userId: 'userId123', + channel: 'mobile', + anonymousId: 'anon_123', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'bloomreach-validation-test-3', + name: destType, + description: 'Missing required field -> timestamp', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Timestamp is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Timestamp is not present. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/router/data.ts b/test/integrations/destinations/bloomreach/router/data.ts new file mode 100644 index 0000000000..e99d0cc8cd --- /dev/null +++ b/test/integrations/destinations/bloomreach/router/data.ts @@ -0,0 +1,220 @@ +import { generateMetadata } from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; +import { + destType, + destination, + traits, + properties, + headers, + endpoint, + RouterInstrumentationErrorStatTags, +} from '../common'; + +const routerRequest = { + input: [ + { + message: { + type: 'track', + anonymousId: 'anonId1', + event: 'test_event_1A', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'identify', + anonymousId: 'anonId1', + userId: 'userId1', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(2), + destination, + }, + { + message: { + type: 'track', + anonymousId: 'anonId2', + event: 'test_event_2A', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(3), + destination, + }, + { + message: { + type: 'track', + anonymousId: 'anonId1', + userId: 'userId1', + event: 'test_event_1B', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(4), + destination, + }, + { + message: { + type: 'identify', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(5), + destination, + }, + ], + destType, +}; +export const data = [ + { + id: 'bloomreach-router-test-1', + name: destType, + description: 'Basic Router Test to test multiple payloads', + scenario: 'Framework', + successCriteria: 'All events should be transformed successfully and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params: {}, + body: { + JSON: { + commands: [ + { + data: { + customer_ids: { cookie: 'anonId1' }, + properties, + timestamp: 1709566376, + event_type: 'test_event_1A', + }, + name: 'customers/events', + }, + { + data: { + customer_ids: { + registered: 'userId1', + cookie: 'anonId1', + }, + properties: { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, + }, + update_timestamp: 1709566376, + }, + name: 'customers', + }, + { + data: { + customer_ids: { cookie: 'anonId2' }, + properties, + timestamp: 1709566376, + event_type: 'test_event_2A', + }, + name: 'customers/events', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1), generateMetadata(2), generateMetadata(3)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params: {}, + body: { + JSON: { + commands: [ + { + data: { + customer_ids: { registered: 'userId1', cookie: 'anonId1' }, + properties, + timestamp: 1709566376, + event_type: 'test_event_1B', + }, + name: 'customers/events', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(4)], + batched: true, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(5)], + batched: false, + statusCode: 400, + error: 'Either one of userId or anonymousId is required. Aborting', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, +]; From 92515a5fd8a2798c48010078f62b360ec6a49979 Mon Sep 17 00:00:00 2001 From: Mihir Bhalala <77438541+mihir-4116@users.noreply.github.com> Date: Mon, 1 Apr 2024 16:57:04 +0530 Subject: [PATCH 4/4] feat: consent field support for ga4 (#3213) * feat: support consent fields in GA4 * chore: code review changes * chore: code review changes * fix(ga4): processor tests --------- Co-authored-by: krishnachaitanya --- src/v0/destinations/ga4/transform.js | 7 + src/v0/destinations/ga4/utils.js | 21 ++ src/v0/destinations/ga4/utils.test.js | 87 ++++- src/v0/util/googleUtils/index.js | 27 ++ src/v0/util/googleUtils/index.test.js | 52 ++- .../destinations/ga4/processor/data.ts | 309 ++++++++++++++++++ 6 files changed, 501 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/ga4/transform.js b/src/v0/destinations/ga4/transform.js index d8fc531e92..5280a46dab 100644 --- a/src/v0/destinations/ga4/transform.js +++ b/src/v0/destinations/ga4/transform.js @@ -27,6 +27,7 @@ const { const { getItemsArray, validateEventName, + prepareUserConsents, removeInvalidParams, isReservedEventName, getGA4ExclusionList, @@ -238,6 +239,12 @@ const responseBuilder = (message, { Config }) => { rawPayload.user_properties = userProperties; } + // Prepare GA4 consents + const consents = prepareUserConsents(message); + if (!isEmptyObject(consents)) { + rawPayload.consent = consents; + } + payload = removeUndefinedAndNullValues(payload); rawPayload = { ...rawPayload, events: [payload] }; diff --git a/src/v0/destinations/ga4/utils.js b/src/v0/destinations/ga4/utils.js index e4db494727..ce8afda560 100644 --- a/src/v0/destinations/ga4/utils.js +++ b/src/v0/destinations/ga4/utils.js @@ -7,8 +7,10 @@ const { isEmptyObject, extractCustomFields, isDefinedAndNotNull, + getIntegrationsObj, } = require('../../util'); const { mappingConfig, ConfigCategory } = require('./config'); +const { finaliseAnalyticsConsents } = require('../../util/googleUtils'); /** * Reserved event names cannot be used @@ -432,11 +434,30 @@ const prepareUserProperties = (message, piiPropertiesToIgnore = []) => { return validatedUserProperties; }; +/** + * Returns user consents + * Ref : https://developers.google.com/analytics/devguides/collection/protocol/ga4/reference?client_type=gtag#payload_consent + * @param {*} message + * @returns + */ +const prepareUserConsents = (message) => { + const integrationObj = getIntegrationsObj(message, 'ga4') || {}; + const eventLevelConsentsData = integrationObj?.consents || {}; + const consentConfigMap = { + analyticsPersonalizationConsent: 'ad_user_data', + analyticsUserDataConsent: 'ad_personalization', + }; + + const consents = finaliseAnalyticsConsents(consentConfigMap, eventLevelConsentsData); + return consents; +}; + module.exports = { getItem, getItemList, getItemsArray, validateEventName, + prepareUserConsents, removeInvalidParams, isReservedEventName, getGA4ExclusionList, diff --git a/src/v0/destinations/ga4/utils.test.js b/src/v0/destinations/ga4/utils.test.js index 18b3ab5766..501778910f 100644 --- a/src/v0/destinations/ga4/utils.test.js +++ b/src/v0/destinations/ga4/utils.test.js @@ -1,4 +1,9 @@ -const { validateEventName, prepareUserProperties, removeInvalidParams } = require('./utils'); +const { + validateEventName, + removeInvalidParams, + prepareUserConsents, + prepareUserProperties, +} = require('./utils'); const userPropertyData = [ { @@ -447,4 +452,84 @@ describe('Google Analytics 4 utils test', () => { expect(result).toEqual(expected); }); }); + + describe('prepareUserConsents function tests', () => { + it('Should return an empty object when no consents are given', () => { + const message = {}; + const result = prepareUserConsents(message); + expect(result).toEqual({}); + }); + + it('Should return an empty object when no consents are given', () => { + const message = { + integrations: { + GA4: {}, + }, + }; + const result = prepareUserConsents(message); + expect(result).toEqual({}); + }); + + it('Should return an empty object when no consents are given', () => { + const message = { + integrations: { + GA4: { + consents: {}, + }, + }, + }; + const result = prepareUserConsents(message); + expect(result).toEqual({}); + }); + + it('Should return a consents object when consents are given', () => { + const message = { + integrations: { + GA4: { + consents: { + ad_personalization: 'GRANTED', + ad_user_data: 'GRANTED', + }, + }, + }, + }; + const result = prepareUserConsents(message); + expect(result).toEqual({ + ad_personalization: 'GRANTED', + ad_user_data: 'GRANTED', + }); + }); + + it('Should return an empty object when invalid consents are given', () => { + const message = { + integrations: { + GA4: { + consents: { + ad_personalization: 'NOT_SPECIFIED', + ad_user_data: 'NOT_SPECIFIED', + }, + }, + }, + }; + const result = prepareUserConsents(message); + expect(result).toEqual({}); + }); + + it('Should return a valid consents values from consents object', () => { + const message = { + integrations: { + GA4: { + consents: { + ad_personalization: 'NOT_SPECIFIED', + ad_user_data: 'DENIED', + }, + }, + }, + }; + const result = prepareUserConsents(message); + expect(result).toEqual({ + ad_user_data: 'DENIED', + }); + }); + }); }); diff --git a/src/v0/util/googleUtils/index.js b/src/v0/util/googleUtils/index.js index c153731e73..ef7c244c17 100644 --- a/src/v0/util/googleUtils/index.js +++ b/src/v0/util/googleUtils/index.js @@ -1,4 +1,5 @@ const GOOGLE_ALLOWED_CONSENT_STATUS = ['UNSPECIFIED', 'UNKNOWN', 'GRANTED', 'DENIED']; +const GA4_ALLOWED_CONSENT_STATUS = ['GRANTED', 'DENIED']; const UNSPECIFIED_CONSENT = 'UNSPECIFIED'; const UNKNOWN_CONSENT = 'UNKNOWN'; @@ -82,10 +83,36 @@ const finaliseConsent = (consentConfigMap, eventLevelConsent = {}, destConfig = return consentObj; }; +/** + * Populates the consent object based on the provided configuration and consent mapping. + * @param {*} consentConfigMap + * @param {*} eventLevelConsent + * @returns + */ +const finaliseAnalyticsConsents = (consentConfigMap, eventLevelConsent = {}) => { + const consentObj = {}; + // Iterate through each key in consentConfigMap to set the consent + Object.keys(consentConfigMap).forEach((configKey) => { + const consentKey = consentConfigMap[configKey]; // e.g., 'ad_user_data' + + // Set consent only if valid + if ( + eventLevelConsent && + eventLevelConsent.hasOwnProperty(consentKey) && + GA4_ALLOWED_CONSENT_STATUS.includes(eventLevelConsent[consentKey]) + ) { + consentObj[consentKey] = eventLevelConsent[consentKey]; + } + }); + + return consentObj; +}; + module.exports = { populateConsentFromConfig, UNSPECIFIED_CONSENT, UNKNOWN_CONSENT, GOOGLE_ALLOWED_CONSENT_STATUS, finaliseConsent, + finaliseAnalyticsConsents, }; diff --git a/src/v0/util/googleUtils/index.test.js b/src/v0/util/googleUtils/index.test.js index 28e0fa9ac8..76ec624311 100644 --- a/src/v0/util/googleUtils/index.test.js +++ b/src/v0/util/googleUtils/index.test.js @@ -1,4 +1,8 @@ -const { populateConsentFromConfig, finaliseConsent } = require('./index'); +const { + finaliseConsent, + populateConsentFromConfig, + finaliseAnalyticsConsents, +} = require('./index'); describe('unit test for populateConsentFromConfig', () => { const consentConfigMap = { @@ -243,3 +247,49 @@ describe('finaliseConsent', () => { }); }); }); + +describe('unit test for finaliseAnalyticsConsents', () => { + const consentConfigMap = { + personalizationConsent: 'ad_personalization', + userDataConsent: 'ad_user_data', + }; + it('Should return an empty object when no valid consents are provided', () => { + const result = finaliseAnalyticsConsents(consentConfigMap, {}); + expect(result).toEqual({}); + }); + + it('Should set ad_user_data property of consent object when userDataConsent property is provided and its value is one of the allowed consent statuses', () => { + const properties = { ad_user_data: 'GRANTED' }; + const result = finaliseAnalyticsConsents(consentConfigMap, properties); + expect(result).toEqual({ ad_user_data: 'GRANTED' }); + }); + + it('Should set ad_personalization property of consent object when personalizationConsent property is provided and its value is one of the allowed consent statuses', () => { + const properties = { ad_personalization: 'DENIED' }; + const result = finaliseAnalyticsConsents(consentConfigMap, properties); + expect(result).toEqual({ ad_personalization: 'DENIED' }); + }); + + it('Should return an empty object when properties parameter is not provided', () => { + const result = finaliseAnalyticsConsents(consentConfigMap, undefined); + expect(result).toEqual({}); + }); + + it('Should return an empty object when properties parameter is null', () => { + const result = finaliseAnalyticsConsents(consentConfigMap, null); + expect(result).toEqual({}); + }); + + it('Should return an empty object when properties parameter is an UNSPECIFIED object', () => { + const result = finaliseAnalyticsConsents(consentConfigMap, {}); + expect(result).toEqual({}); + }); + + it('should return empty object when properties parameter contains ad_user_data and ad_personalization with non-allowed values', () => { + const result = finaliseAnalyticsConsents(consentConfigMap, { + userDataConsent: 'RANDOM', + personalizationConsent: 'RANDOM', + }); + expect(result).toEqual({}); + }); +}); diff --git a/test/integrations/destinations/ga4/processor/data.ts b/test/integrations/destinations/ga4/processor/data.ts index f96ca9e74a..4465ec9e2c 100644 --- a/test/integrations/destinations/ga4/processor/data.ts +++ b/test/integrations/destinations/ga4/processor/data.ts @@ -14900,4 +14900,313 @@ export const data = [ }, mockFns: defaultMockFns, }, + { + name: 'ga4', + description: '(gtag) send consents setting to ga4 with login event', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + messageId: 'ec5481b6-a926-4d2e-b293-0b3a77c4d3be', + originalTimestamp: '2022-04-26T05:17:09Z', + anonymousId: 'ea5cfab2-3961-4d8a-8187-3d1858c99090', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + adTrackingEnabled: 'false', + advertisingId: 'T0T0T072-5e28-45a1-9eda-ce22a3e36d1a', + id: '3f034872-5e28-45a1-9eda-ce22a3e36d1a', + manufacturer: 'Google', + model: 'AOSP on IA Emulator', + name: 'generic_x86_arm', + type: 'ios', + attTrackingStatus: 3, + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + locale: 'en-US', + os: { + name: 'iOS', + version: '14.4.1', + }, + screen: { + density: 2, + }, + externalId: [ + { + type: 'ga4AppInstanceId', + id: 'dummyGA4AppInstanceId', + }, + { + type: 'ga4ClientId', + id: 'client_id', + }, + ], + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', + }, + type: 'track', + event: 'login', + properties: { + method: 'Google', + }, + integrations: { + All: true, + GA4: { + consents: { + ad_personalization: 'GRANTED', + ad_user_data: 'GRANTED', + }, + }, + }, + sentAt: '2022-04-20T15:20:57Z', + }, + destination: { + Config: { + apiSecret: 'dummyApiSecret', + measurementId: 'G-123456', + firebaseAppId: '', + blockPageViewEvent: false, + typesOfClient: 'gtag', + extendPageViewParams: false, + sendUserId: false, + eventFilteringOption: 'disable', + blacklistedEvents: [ + { + eventName: '', + }, + ], + whitelistedEvents: [ + { + eventName: '', + }, + ], + enableServerSideIdentify: false, + sendLoginSignup: false, + generateLead: false, + }, + Enabled: true, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.google-analytics.com/mp/collect', + headers: { + HOST: 'www.google-analytics.com', + 'Content-Type': 'application/json', + }, + params: { + api_secret: 'dummyApiSecret', + measurement_id: 'G-123456', + }, + body: { + JSON: { + client_id: 'client_id', + consent: { + ad_personalization: 'GRANTED', + ad_user_data: 'GRANTED', + }, + timestamp_micros: 1650950229000000, + non_personalized_ads: true, + events: [ + { + name: 'login', + params: { + method: 'Google', + engagement_time_msec: 1, + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + name: 'ga4', + description: '(gtag) send consents setting to ga4 with login event', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + messageId: 'ec5481b6-a926-4d2e-b293-0b3a77c4d3be', + originalTimestamp: '2022-04-26T05:17:09Z', + anonymousId: 'ea5cfab2-3961-4d8a-8187-3d1858c99090', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + adTrackingEnabled: 'false', + advertisingId: 'T0T0T072-5e28-45a1-9eda-ce22a3e36d1a', + id: '3f034872-5e28-45a1-9eda-ce22a3e36d1a', + manufacturer: 'Google', + model: 'AOSP on IA Emulator', + name: 'generic_x86_arm', + type: 'ios', + attTrackingStatus: 3, + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + locale: 'en-US', + os: { + name: 'iOS', + version: '14.4.1', + }, + screen: { + density: 2, + }, + externalId: [ + { + type: 'ga4AppInstanceId', + id: 'dummyGA4AppInstanceId', + }, + { + type: 'ga4ClientId', + id: 'client_id', + }, + ], + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', + }, + type: 'track', + event: 'login', + properties: { + method: 'Google', + }, + integrations: { + All: true, + GA4: { + consents: { + ad_personalization: 'NOT_SPECIFIED', + ad_user_data: 'DENIED', + }, + }, + }, + sentAt: '2022-04-20T15:20:57Z', + }, + destination: { + Config: { + apiSecret: 'dummyApiSecret', + measurementId: 'G-123456', + firebaseAppId: '', + blockPageViewEvent: false, + typesOfClient: 'gtag', + extendPageViewParams: false, + sendUserId: false, + eventFilteringOption: 'disable', + blacklistedEvents: [ + { + eventName: '', + }, + ], + whitelistedEvents: [ + { + eventName: '', + }, + ], + enableServerSideIdentify: false, + sendLoginSignup: false, + generateLead: false, + }, + Enabled: true, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.google-analytics.com/mp/collect', + headers: { + HOST: 'www.google-analytics.com', + 'Content-Type': 'application/json', + }, + params: { + api_secret: 'dummyApiSecret', + measurement_id: 'G-123456', + }, + body: { + JSON: { + client_id: 'client_id', + consent: { + ad_user_data: 'DENIED', + }, + timestamp_micros: 1650950229000000, + non_personalized_ads: true, + events: [ + { + name: 'login', + params: { + method: 'Google', + engagement_time_msec: 1, + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, ];