diff --git a/src/cdk/v2/destinations/movable_ink/config.js b/src/cdk/v2/destinations/movable_ink/config.js new file mode 100644 index 0000000000..673e94620e --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/config.js @@ -0,0 +1,3 @@ +module.exports = { + MAX_REQUEST_SIZE_IN_BYTES: 13500, +}; diff --git a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml new file mode 100644 index 0000000000..25270058c5 --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml @@ -0,0 +1,72 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: defaultRequestConfig + path: ../../../../v0/util + - name: toUnixTimestampInMS + path: ../../../../v0/util + - name: base64Convertor + path: ../../../../v0/util + - path: ./utils + +steps: + - name: messageType + template: | + .message.type.toLowerCase(); + + - name: validateInput + template: | + let messageType = $.outputs.messageType; + $.assert(messageType, "message Type is not present. Aborting"); + $.assert(messageType in {{$.EventType.([.IDENTIFY,.TRACK])}}, "message type " + messageType + " is not supported"); + $.assertConfig(.destination.Config.endpoint, "Movable Ink Endpoint is not present. Aborting"); + $.assertConfig(.destination.Config.accessKey, "Access key is not present . Aborting"); + $.assertConfig(.destination.Config.accessSecret, "Access Secret is not present. Aborting"); + $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting"); + + const userId = .message.().( + {{{{$.getGenericPaths("userIdOnly")}}}}; + ); + const email = .message.().( + {{{{$.getGenericPaths("email")}}}}; + ); + + $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting"); + + - name: preparePayload + description: Prepare payload for identify and track. This payload schema needs to be configured in the Movable Ink dashboard. Movable Ink will discard any additional fields from the input payload. + template: | + const userId = .message.().( + {{{{$.getGenericPaths("userIdOnly")}}}}; + ); + const email = .message.().( + {{{{$.getGenericPaths("email")}}}}; + ); + const timestampInUnix = $.toUnixTimestampInMS(.message.().( + {{{{$.getGenericPaths("timestamp")}}}}; + )); + $.context.payload = { + ...(.message), + userId: userId ?? email, + timestamp: timestampInUnix, + anonymousId: .message.anonymousId + } + + - name: buildResponse + description: In batchMode we return payload directly + condition: $.batchMode + template: | + $.context.payload + else: + name: buildResponseForProcessTransformation + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = $.context.payload; + response.endpoint = .destination.Config.endpoint; + response.method = "POST"; + response.headers = { + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.destination.Config.accessKey + ":" + .destination.Config.accessSecret) + } + response; diff --git a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml new file mode 100644 index 0000000000..46afb34d53 --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml @@ -0,0 +1,74 @@ +bindings: + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - path: ./utils + exportAll: true + - name: base64Convertor + path: ../../../../v0/util + - name: BatchUtils + path: '@rudderstack/workflow-engine' + - path: ./config + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + bindings: + - name: batchMode + value: true + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "batchedRequest": ., + "batched": false, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata, + "statusCode": 200 + })[] + + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + let batches = $.BatchUtils.chunkArrayBySizeAndLength( + $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES}).items; + + batches@batch.({ + "batchedRequest": { + "body": { + "JSON": {"events": ~r batch.batchedRequest[]}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": batch[0].destination.Config.().(.endpoint), + "headers": batch[0].destination.Config.().({ + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.accessKey + ":" + .accessSecret) + }), + "params": {}, + "files": {} + }, + "metadata": ~r batch.metadata[], + "batched": true, + "statusCode": 200, + "destination": batch[0].destination + })[]; + + - name: finalPayload + template: | + [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents] diff --git a/src/features.json b/src/features.json index dc52044048..76b562a825 100644 --- a/src/features.json +++ b/src/features.json @@ -67,6 +67,7 @@ "THE_TRADE_DESK": true, "INTERCOM": true, "NINETAILED": true, + "MOVABLE_INK": true, "KOALA": true }, "regulations": [ diff --git a/test/integrations/destinations/movable_ink/common.ts b/test/integrations/destinations/movable_ink/common.ts new file mode 100644 index 0000000000..f7eaa7af39 --- /dev/null +++ b/test/integrations/destinations/movable_ink/common.ts @@ -0,0 +1,128 @@ +import { Destination } from '../../../../src/types'; + +const destType = 'movable_ink'; +const destTypeInUpperCase = 'MOVABLE_INK'; +const displayName = 'Movable Ink'; +const channel = 'web'; +const destination: Destination = { + Config: { + endpoint: 'https://collector.movableink-dmz.com/behavioral/abc123', + accessKey: 'test-access-key', + accessSecret: 'test_access_secret', + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', +}; + +const processorInstrumentationErrorStatTags = { + destType: destTypeInUpperCase, + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', +}; + +const RouterInstrumentationErrorStatTags = { + ...processorInstrumentationErrorStatTags, + feature: 'router', +}; + +const traits = { + email: 'test@example.com', + firstName: 'John', + lastName: 'Doe', + phone: '1234567890', +}; + +const headers = { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC1hY2Nlc3Mta2V5OnRlc3RfYWNjZXNzX3NlY3JldA==', +}; + +const commonProperties = { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + categories: [ + { url: 'https://example1', id: '1' }, + { url: 'https://example2', id: '2' }, + ], + name: 'Cones of Dunshire', + brand: 'Wyatt Games', + variant: 'expansion pack', + price: 49.99, + quantity: 5, + coupon: 'PREORDER15', + position: 1, + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.webp', +}; + +const customProperties = { + key1: 'value1', + key2: true, + key3: ['value3'], + key4: { key5: { key6: 'value6' } }, +}; + +const trackTestProperties = { + 'Product Added': { ...commonProperties, ...customProperties }, + 'Product Viewed': { ...commonProperties, ...customProperties }, + 'Order Completed': { + checkout_id: '70324a1f0eaf000000000000', + order_id: '40684e8f0eaf000000000000', + affiliation: 'Vandelay Games', + total: 52, + subtotal: 45, + revenue: 50, + shipping: 4, + tax: 3, + discount: 5, + coupon: 'NEWCUST5', + currency: 'USD', + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + name: 'Cones of Dunshire', + price: 40, + position: 1, + category: 'Games', + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.jpg', + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + name: 'Five Crowns', + price: 5, + position: 2, + category: 'Games', + }, + ], + }, + 'Products Searched': { query: 'HDMI cable', url: 'https://www.website.com/product/path' }, + 'Custom event': { ...commonProperties, key1: 'value1', key2: true }, +}; + +export { + destType, + channel, + destination, + processorInstrumentationErrorStatTags, + RouterInstrumentationErrorStatTags, + traits, + headers, + trackTestProperties, +}; diff --git a/test/integrations/destinations/movable_ink/processor/data.ts b/test/integrations/destinations/movable_ink/processor/data.ts new file mode 100644 index 0000000000..45453c74cd --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/data.ts @@ -0,0 +1,4 @@ +import { validation } from './validation'; +import { identify } from './identify'; +import { track } from './track'; +export const data = [...identify, ...track, ...validation]; diff --git a/test/integrations/destinations/movable_ink/processor/identify.ts b/test/integrations/destinations/movable_ink/processor/identify.ts new file mode 100644 index 0000000000..27186da05c --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/identify.ts @@ -0,0 +1,64 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, channel, destination, traits, headers } from '../common'; + +export const identify: ProcessorTestData[] = [ + { + id: 'MovableInk-identify-test-1', + name: destType, + description: 'Identify call with traits and anonymousId', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'identify', + userId: traits.email, + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/processor/track.ts b/test/integrations/destinations/movable_ink/processor/track.ts new file mode 100644 index 0000000000..5f30a3de83 --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/track.ts @@ -0,0 +1,189 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, channel, destination, headers, trackTestProperties } from '../common'; + +export const track: ProcessorTestData[] = [ + { + id: 'MovableInk-track-test-1', + name: destType, + description: 'Track call: Product Added event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'MovableInk-track-test-2', + name: destType, + description: 'Track call: Order Completed event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'MovableInk-track-test-3', + name: destType, + description: 'Track call: Custom event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/processor/validation.ts b/test/integrations/destinations/movable_ink/processor/validation.ts new file mode 100644 index 0000000000..f9f6c6a927 --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/validation.ts @@ -0,0 +1,131 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata } from '../../../testUtils'; +import { destType, destination, processorInstrumentationErrorStatTags } from '../common'; + +export const validation: ProcessorTestData[] = [ + { + id: 'MovableInk-validation-test-1', + name: destType, + description: 'All of the required fields — userId, email, and anonymousId — are missing.', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Either one of userId or email or anonymousId is required. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Either one of userId or email or anonymousId is required. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'MovableInk-validation-test-2', + name: destType, + description: 'Unsupported message type -> group', + scenario: 'Framework', + successCriteria: 'Instrumentation Error for Unsupported message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'group', + userId: 'userId123', + channel: 'mobile', + anonymousId: 'anon_123', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'MovableInk-validation-test-3', + name: destType, + description: 'Missing required field -> timestamp', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Timestamp is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Timestamp is not present. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/router/data.ts b/test/integrations/destinations/movable_ink/router/data.ts new file mode 100644 index 0000000000..72df3d7074 --- /dev/null +++ b/test/integrations/destinations/movable_ink/router/data.ts @@ -0,0 +1,162 @@ +import { RouterTestData } from '../../../testTypes'; +import { RouterTransformationRequest } from '../../../../../src/types'; +import { generateMetadata } from '../../../testUtils'; +import { + destType, + channel, + destination, + traits, + headers, + trackTestProperties, + RouterInstrumentationErrorStatTags, +} from '../common'; + +const routerRequest: RouterTransformationRequest = { + input: [ + { + message: { + type: 'identify', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(2), + destination, + }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(3), + destination, + }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + }, + metadata: generateMetadata(4), + destination, + }, + ], + destType, +}; + +export const data: RouterTestData[] = [ + { + id: 'MovableInk-router-test-1', + name: destType, + description: 'Basic Router Test to test multiple payloads', + scenario: 'Framework', + successCriteria: + 'Some events should be transformed successfully and some should fail for missing fields and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + events: [ + { + type: 'identify', + userId: traits.email, + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1), generateMetadata(3)], + batched: true, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(2)], + batched: false, + statusCode: 400, + error: 'Either one of userId or email or anonymousId is required. Aborting', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(4)], + batched: false, + statusCode: 400, + error: 'Timestamp is not present. Aborting', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + ], + }, + }, + }, + }, +];