From ca8d6953f386a72f66c0bdd8ad926e6108c1eafb Mon Sep 17 00:00:00 2001 From: Gauravudia Date: Tue, 5 Mar 2024 12:57:06 +0530 Subject: [PATCH 1/5] feat: onboard destination movable ink --- .../movable_ink/procWorkflow.yaml | 65 ++++++ .../destinations/movable_ink/rtWorkflow.yaml | 33 +++ src/features.json | 3 +- .../destinations/movable_ink/common.ts | 121 +++++++++++ .../movable_ink/processor/data.ts | 4 + .../movable_ink/processor/identify.ts | 64 ++++++ .../movable_ink/processor/track.ts | 189 ++++++++++++++++++ .../movable_ink/processor/validation.ts | 131 ++++++++++++ .../destinations/movable_ink/router/data.ts | 178 +++++++++++++++++ 9 files changed, 787 insertions(+), 1 deletion(-) create mode 100644 src/cdk/v2/destinations/movable_ink/procWorkflow.yaml create mode 100644 src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml create mode 100644 test/integrations/destinations/movable_ink/common.ts create mode 100644 test/integrations/destinations/movable_ink/processor/data.ts create mode 100644 test/integrations/destinations/movable_ink/processor/identify.ts create mode 100644 test/integrations/destinations/movable_ink/processor/track.ts create mode 100644 test/integrations/destinations/movable_ink/processor/validation.ts create mode 100644 test/integrations/destinations/movable_ink/router/data.ts diff --git a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml new file mode 100644 index 0000000000..93e0c8699d --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml @@ -0,0 +1,65 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: defaultRequestConfig + path: ../../../../v0/util + - name: toUnixTimestampInMS + path: ../../../../v0/util + - name: base64Convertor + path: ../../../../v0/util + - path: ./utils + +steps: + - name: messageType + template: | + .message.type.toLowerCase(); + + - name: validateInput + template: | + let messageType = $.outputs.messageType; + $.assert(messageType, "message Type is not present. Aborting"); + $.assert(messageType in {{$.EventType.([.IDENTIFY,.TRACK])}}, "message type " + messageType + " is not supported"); + $.assertConfig(.destination.Config.endpoint, "Movable Ink Endpoint is not present. Aborting"); + $.assertConfig(.destination.Config.accessKey, "Access key is not present . Aborting"); + $.assertConfig(.destination.Config.accessSecret, "Access Secret is not present. Aborting"); + $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting"); + + const userId = .message.().( + {{{{$.getGenericPaths("userIdOnly")}}}}; + ); + const email = .message.().( + {{{{$.getGenericPaths("email")}}}}; + ); + + $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting"); + + - name: preparePayload + template: | + const userId = .message.().( + {{{{$.getGenericPaths("userIdOnly")}}}}; + ); + const email = .message.().( + {{{{$.getGenericPaths("email")}}}}; + ); + const timestampInUnix = $.toUnixTimestampInMS(.message.().( + {{{{$.getGenericPaths("timestamp")}}}}; + )); + $.context.payload = { + ...(.message), + userId: userId ?? email, + timestamp: timestampInUnix, + anonymousId: .message.anonymousId + } + + - name: buildResponse + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = $.context.payload; + response.endpoint = .destination.Config.endpoint; + response.method = "POST"; + response.headers = { + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.destination.Config.accessKey + ":" + .destination.Config.accessSecret) + } + response; diff --git a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml new file mode 100644 index 0000000000..fc5d474d60 --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml @@ -0,0 +1,33 @@ +bindings: + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - path: ./utils + exportAll: true + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "batchedRequest": ., + "batched": false, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata[], + "statusCode": 200 + })[] + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + - name: finalPayload + template: | + [...$.outputs.successfulEvents, ...$.outputs.failedEvents] diff --git a/src/features.json b/src/features.json index 5460111a22..bbb92dd20d 100644 --- a/src/features.json +++ b/src/features.json @@ -66,7 +66,8 @@ "REDDIT": true, "THE_TRADE_DESK": true, "INTERCOM": true, - "NINETAILED": true + "NINETAILED": true, + "MOVABLE_INK": true }, "regulations": [ "BRAZE", diff --git a/test/integrations/destinations/movable_ink/common.ts b/test/integrations/destinations/movable_ink/common.ts new file mode 100644 index 0000000000..6744bdb396 --- /dev/null +++ b/test/integrations/destinations/movable_ink/common.ts @@ -0,0 +1,121 @@ +import { Destination } from '../../../../src/types'; + +const destType = 'movable_ink'; +const destTypeInUpperCase = 'MOVABLE_INK'; +const displayName = 'Movable Ink'; +const channel = 'web'; +const destination: Destination = { + Config: { + endpoint: 'https://collector.movableink-dmz.com/behavioral/abc123', + accessKey: 'test-access-key', + accessSecret: 'test_access_secret', + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', +}; + +const processorInstrumentationErrorStatTags = { + destType: destTypeInUpperCase, + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', +}; + +const RouterInstrumentationErrorStatTags = { + ...processorInstrumentationErrorStatTags, + feature: 'router', +}; + +const traits = { + email: 'test@example.com', + firstName: 'John', + lastName: 'Doe', + phone: '1234567890', +}; + +const headers = { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC1hY2Nlc3Mta2V5OnRlc3RfYWNjZXNzX3NlY3JldA==', +}; + +const commonProperties = { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + categories: [ + { url: 'https://example1', id: '1' }, + { url: 'https://example2', id: '2' }, + ], + name: 'Cones of Dunshire', + brand: 'Wyatt Games', + variant: 'expansion pack', + price: 49.99, + quantity: 5, + coupon: 'PREORDER15', + position: 1, + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.webp', +}; + +const trackTestProperties = { + 'Product Added': commonProperties, + 'Product Viewed': commonProperties, + 'Order Completed': { + checkout_id: '70324a1f0eaf000000000000', + order_id: '40684e8f0eaf000000000000', + affiliation: 'Vandelay Games', + total: 52, + subtotal: 45, + revenue: 50, + shipping: 4, + tax: 3, + discount: 5, + coupon: 'NEWCUST5', + currency: 'USD', + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + name: 'Cones of Dunshire', + price: 40, + position: 1, + category: 'Games', + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.jpg', + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + name: 'Five Crowns', + price: 5, + position: 2, + category: 'Games', + }, + ], + }, + 'Products Searched': { query: 'HDMI cable', url: 'https://www.website.com/product/path' }, + 'Custom event': { ...commonProperties, key1: 'value1', key2: true }, +}; + +export { + destType, + channel, + destination, + processorInstrumentationErrorStatTags, + RouterInstrumentationErrorStatTags, + traits, + headers, + trackTestProperties, +}; diff --git a/test/integrations/destinations/movable_ink/processor/data.ts b/test/integrations/destinations/movable_ink/processor/data.ts new file mode 100644 index 0000000000..45453c74cd --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/data.ts @@ -0,0 +1,4 @@ +import { validation } from './validation'; +import { identify } from './identify'; +import { track } from './track'; +export const data = [...identify, ...track, ...validation]; diff --git a/test/integrations/destinations/movable_ink/processor/identify.ts b/test/integrations/destinations/movable_ink/processor/identify.ts new file mode 100644 index 0000000000..27186da05c --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/identify.ts @@ -0,0 +1,64 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, channel, destination, traits, headers } from '../common'; + +export const identify: ProcessorTestData[] = [ + { + id: 'MovableInk-identify-test-1', + name: destType, + description: 'Identify call with traits and anonymousId', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'identify', + userId: traits.email, + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/processor/track.ts b/test/integrations/destinations/movable_ink/processor/track.ts new file mode 100644 index 0000000000..5f30a3de83 --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/track.ts @@ -0,0 +1,189 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, channel, destination, headers, trackTestProperties } from '../common'; + +export const track: ProcessorTestData[] = [ + { + id: 'MovableInk-track-test-1', + name: destType, + description: 'Track call: Product Added event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'MovableInk-track-test-2', + name: destType, + description: 'Track call: Order Completed event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'MovableInk-track-test-3', + name: destType, + description: 'Track call: Custom event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/processor/validation.ts b/test/integrations/destinations/movable_ink/processor/validation.ts new file mode 100644 index 0000000000..f9f6c6a927 --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/validation.ts @@ -0,0 +1,131 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata } from '../../../testUtils'; +import { destType, destination, processorInstrumentationErrorStatTags } from '../common'; + +export const validation: ProcessorTestData[] = [ + { + id: 'MovableInk-validation-test-1', + name: destType, + description: 'All of the required fields — userId, email, and anonymousId — are missing.', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Either one of userId or email or anonymousId is required. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Either one of userId or email or anonymousId is required. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'MovableInk-validation-test-2', + name: destType, + description: 'Unsupported message type -> group', + scenario: 'Framework', + successCriteria: 'Instrumentation Error for Unsupported message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'group', + userId: 'userId123', + channel: 'mobile', + anonymousId: 'anon_123', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'MovableInk-validation-test-3', + name: destType, + description: 'Missing required field -> timestamp', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Timestamp is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Timestamp is not present. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/router/data.ts b/test/integrations/destinations/movable_ink/router/data.ts new file mode 100644 index 0000000000..fcb4324e7b --- /dev/null +++ b/test/integrations/destinations/movable_ink/router/data.ts @@ -0,0 +1,178 @@ +import { RouterTestData } from '../../../testTypes'; +import { RouterTransformationRequest } from '../../../../../src/types'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { + destType, + channel, + destination, + traits, + headers, + trackTestProperties, + RouterInstrumentationErrorStatTags, +} from '../common'; + +const routerRequest: RouterTransformationRequest = { + input: [ + { + message: { + type: 'identify', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(2), + destination, + }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(3), + destination, + }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + }, + metadata: generateMetadata(4), + destination, + }, + ], + destType, +}; + +export const data: RouterTestData[] = [ + { + id: 'MovableInk-router-test-1', + name: destType, + description: 'Basic Router Test to test multiple payloads', + scenario: 'Framework', + successCriteria: + 'Some events should be transformed successfully and some should fail for missing fields and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + type: 'identify', + userId: traits.email, + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1)], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(3)], + batched: false, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(2)], + batched: false, + statusCode: 400, + error: 'Either one of userId or email or anonymousId is required. Aborting', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(4)], + batched: false, + statusCode: 400, + error: 'Timestamp is not present. Aborting', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + ], + }, + }, + }, + }, +]; From df4fb1cffb61cec7f770b8b95862ba446995dec9 Mon Sep 17 00:00:00 2001 From: Gauravudia Date: Tue, 5 Mar 2024 13:11:40 +0530 Subject: [PATCH 2/5] test: update common.ts --- test/integrations/destinations/movable_ink/common.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/integrations/destinations/movable_ink/common.ts b/test/integrations/destinations/movable_ink/common.ts index 6744bdb396..f7eaa7af39 100644 --- a/test/integrations/destinations/movable_ink/common.ts +++ b/test/integrations/destinations/movable_ink/common.ts @@ -69,9 +69,16 @@ const commonProperties = { image_url: 'https://www.website.com/product/path.webp', }; +const customProperties = { + key1: 'value1', + key2: true, + key3: ['value3'], + key4: { key5: { key6: 'value6' } }, +}; + const trackTestProperties = { - 'Product Added': commonProperties, - 'Product Viewed': commonProperties, + 'Product Added': { ...commonProperties, ...customProperties }, + 'Product Viewed': { ...commonProperties, ...customProperties }, 'Order Completed': { checkout_id: '70324a1f0eaf000000000000', order_id: '40684e8f0eaf000000000000', From 4199b02bc62ad1a5b998327ca4231fa6f8d3a732 Mon Sep 17 00:00:00 2001 From: Gauravudia Date: Mon, 11 Mar 2024 12:56:16 +0530 Subject: [PATCH 3/5] feat: add batching support --- .../movable_ink/procWorkflow.yaml | 24 ++++--- .../destinations/movable_ink/rtWorkflow.yaml | 40 ++++++++++- .../destinations/movable_ink/router/data.ts | 72 ++++++++----------- 3 files changed, 81 insertions(+), 55 deletions(-) diff --git a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml index 93e0c8699d..200e94b866 100644 --- a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml +++ b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml @@ -53,13 +53,19 @@ steps: } - name: buildResponse + description: In batchMode we return payload directly + condition: $.batchMode template: | - const response = $.defaultRequestConfig(); - response.body.JSON = $.context.payload; - response.endpoint = .destination.Config.endpoint; - response.method = "POST"; - response.headers = { - "Content-Type": "application/json", - "Authorization": "Basic " + $.base64Convertor(.destination.Config.accessKey + ":" + .destination.Config.accessSecret) - } - response; + $.context.payload + else: + name: buildResponseForProcessTransformation + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = $.context.payload; + response.endpoint = .destination.Config.endpoint; + response.method = "POST"; + response.headers = { + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.destination.Config.accessKey + ":" + .destination.Config.accessSecret) + } + response; diff --git a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml index fc5d474d60..57670cab09 100644 --- a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml +++ b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml @@ -3,6 +3,8 @@ bindings: path: ../../../../v0/util/index - path: ./utils exportAll: true + - name: base64Convertor + path: ../../../../v0/util steps: - name: validateInput @@ -12,6 +14,9 @@ steps: - name: transform externalWorkflow: path: ./procWorkflow.yaml + bindings: + - name: batchMode + value: true loopOverInput: true - name: successfulEvents @@ -20,14 +25,45 @@ steps: "batchedRequest": ., "batched": false, "destination": ^[idx].destination, - "metadata": ^[idx].metadata[], + "metadata": ^[idx].metadata, "statusCode": 200 })[] + - name: failedEvents template: | $.outputs.transform#idx.error.( $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) )[] + + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + let batches = [$.outputs.successfulEvents]; + batches@batch.({ + "batchedRequest": { + "body": { + "JSON": {"events": ~r batch.batchedRequest[]}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": batch[0].destination.Config.().(.endpoint), + "headers": batch[0].destination.Config.().({ + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.accessKey + ":" + .accessSecret) + }), + "params": {}, + "files": {} + }, + "metadata": ~r batch.metadata[], + "batched": true, + "statusCode": 200, + "destination": batch[0].destination + })[]; + - name: finalPayload template: | - [...$.outputs.successfulEvents, ...$.outputs.failedEvents] + [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents] diff --git a/test/integrations/destinations/movable_ink/router/data.ts b/test/integrations/destinations/movable_ink/router/data.ts index fcb4324e7b..72df3d7074 100644 --- a/test/integrations/destinations/movable_ink/router/data.ts +++ b/test/integrations/destinations/movable_ink/router/data.ts @@ -1,6 +1,6 @@ import { RouterTestData } from '../../../testTypes'; import { RouterTransformationRequest } from '../../../../../src/types'; -import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { generateMetadata } from '../../../testUtils'; import { destType, channel, @@ -101,15 +101,31 @@ export const data: RouterTestData[] = [ params: {}, body: { JSON: { - type: 'identify', - userId: traits.email, - anonymousId: 'anonId123', - traits, - integrations: { - All: true, - }, - originalTimestamp: '2024-03-04T15:32:56.409Z', - timestamp: 1709566376409, + events: [ + { + type: 'identify', + userId: traits.email, + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + ], }, JSON_ARRAY: {}, XML: {}, @@ -117,40 +133,8 @@ export const data: RouterTestData[] = [ }, files: {}, }, - metadata: [generateMetadata(1)], - batched: false, - statusCode: 200, - destination, - }, - { - batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: destination.Config.endpoint, - headers, - params: {}, - body: { - JSON: { - type: 'track', - channel, - userId: 'userId123', - anonymousId: 'anonId123', - properties: trackTestProperties['Product Added'], - integrations: { - All: true, - }, - originalTimestamp: '2024-03-04T15:32:56.409Z', - timestamp: 1709566376409, - }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, - }, - files: {}, - }, - metadata: [generateMetadata(3)], - batched: false, + metadata: [generateMetadata(1), generateMetadata(3)], + batched: true, statusCode: 200, destination, }, From a7f43cc7d6895610d2b3fbb55e8e06d46d3e6495 Mon Sep 17 00:00:00 2001 From: Gauravudia Date: Thu, 14 Mar 2024 10:58:34 +0530 Subject: [PATCH 4/5] feat: batching on max size in bytes --- src/cdk/v2/destinations/movable_ink/config.js | 3 +++ src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 src/cdk/v2/destinations/movable_ink/config.js diff --git a/src/cdk/v2/destinations/movable_ink/config.js b/src/cdk/v2/destinations/movable_ink/config.js new file mode 100644 index 0000000000..673e94620e --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/config.js @@ -0,0 +1,3 @@ +module.exports = { + MAX_REQUEST_SIZE_IN_BYTES: 13500, +}; diff --git a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml index 57670cab09..46afb34d53 100644 --- a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml +++ b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml @@ -5,6 +5,9 @@ bindings: exportAll: true - name: base64Convertor path: ../../../../v0/util + - name: BatchUtils + path: '@rudderstack/workflow-engine' + - path: ./config steps: - name: validateInput @@ -38,7 +41,9 @@ steps: - name: batchSuccessfulEvents description: Batches the successfulEvents template: | - let batches = [$.outputs.successfulEvents]; + let batches = $.BatchUtils.chunkArrayBySizeAndLength( + $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES}).items; + batches@batch.({ "batchedRequest": { "body": { From 39406e71fe66b3c11d451c4fec1983dd86b4a13b Mon Sep 17 00:00:00 2001 From: Gauravudia Date: Fri, 15 Mar 2024 12:10:38 +0530 Subject: [PATCH 5/5] docs: added comments --- src/cdk/v2/destinations/movable_ink/procWorkflow.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml index 200e94b866..25270058c5 100644 --- a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml +++ b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml @@ -35,6 +35,7 @@ steps: $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting"); - name: preparePayload + description: Prepare payload for identify and track. This payload schema needs to be configured in the Movable Ink dashboard. Movable Ink will discard any additional fields from the input payload. template: | const userId = .message.().( {{{{$.getGenericPaths("userIdOnly")}}}};