From c0ad21463981ef66154c8157083924f76825762d Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Wed, 13 Mar 2024 22:09:43 +0530 Subject: [PATCH 1/8] feat: add support of --- src/v0/destinations/am/transform.js | 3 ++ src/v0/destinations/am/util.test.js | 35 ++++++++++++++++++- src/v0/destinations/am/utils.js | 9 +++++ .../destinations/am/processor/data.ts | 3 ++ 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/v0/destinations/am/transform.js b/src/v0/destinations/am/transform.js index 2d78479ced..ce157e7674 100644 --- a/src/v0/destinations/am/transform.js +++ b/src/v0/destinations/am/transform.js @@ -525,6 +525,9 @@ const responseBuilderSimple = ( ...campaign, }; + // we are updating the payload with skip_user_properties_sync + AMUtils.updateWithSkipAttribute(message, rawPayload); + const respData = getResponseData(evType, destination, rawPayload, message, groupInfo); const { groups, rawPayload: updatedRawPayload } = respData; diff --git a/src/v0/destinations/am/util.test.js b/src/v0/destinations/am/util.test.js index 723ff3a302..498980d182 100644 --- a/src/v0/destinations/am/util.test.js +++ b/src/v0/destinations/am/util.test.js @@ -1,4 +1,9 @@ -const { getUnsetObj, validateEventType, userPropertiesPostProcess } = require('./utils'); +const { + getUnsetObj, + validateEventType, + userPropertiesPostProcess, + updateWithSkipAttribute, +} = require('./utils'); describe('getUnsetObj', () => { it("should return undefined when 'message.integrations.Amplitude.fieldsToUnset' is not array", () => { @@ -164,3 +169,31 @@ describe('userPropertiesPostProcess', () => { }); }); }); + +describe('updateWithSkipAttribute', () => { + // when 'skipUserPropertiesSync ' is present in 'integrations.Amplitude', return the original payload. + it("should return the original payload when 'skipUserPropertiesSync' is present", () => { + const message = { integrations: { Amplitude: { skipUserPropertiesSync: true } } }; + const payload = { key: 'value' }; + const expectedPayload = { key: 'value', $skip_user_properties_sync: true }; + updateWithSkipAttribute(message, payload); + expect(expectedPayload).toEqual(payload); + }); + + // When 'skipUserPropertiesSync' is not present in 'integrations.Amplitude', return the original payload. + it("should return the original payload when 'skipUserPropertiesSync' is not present", () => { + const message = { integrations: { Amplitude: {} } }; + const payload = { key: 'value' }; + const expectedPayload = { key: 'value' }; + updateWithSkipAttribute(message, payload); + expect(payload).toEqual(expectedPayload); + }); + // When 'message' is null, return null. + it("should return null when 'message' is null", () => { + const message = null; + const payload = { key: 'value' }; + const expectedPayload = { key: 'value' }; + updateWithSkipAttribute(message, payload); + expect(payload).toEqual(expectedPayload); + }); +}); diff --git a/src/v0/destinations/am/utils.js b/src/v0/destinations/am/utils.js index 190a5c1bae..8de899182b 100644 --- a/src/v0/destinations/am/utils.js +++ b/src/v0/destinations/am/utils.js @@ -11,6 +11,7 @@ const get = require('get-value'); const uaParser = require('@amplitude/ua-parser-js'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const set = require('set-value'); const logger = require('../../../logger'); const { isDefinedAndNotNull } = require('../../util'); @@ -110,6 +111,13 @@ const getUnsetObj = (message) => { return unsetObject; }; +const updateWithSkipAttribute = (message, payload) => { + const skipAttribute = get(message, 'integrations.Amplitude.skipUserPropertiesSync'); + if (skipAttribute) { + set(payload, '$skip_user_properties_sync', true); + } +}; + /** * Check for evType as in some cases, like when the page name is absent, * either the template depends only on the event.name or there is no template provided by user @@ -187,4 +195,5 @@ module.exports = { getUnsetObj, validateEventType, userPropertiesPostProcess, + updateWithSkipAttribute, }; diff --git a/test/integrations/destinations/am/processor/data.ts b/test/integrations/destinations/am/processor/data.ts index b645fb5ac7..01f9feb44a 100644 --- a/test/integrations/destinations/am/processor/data.ts +++ b/test/integrations/destinations/am/processor/data.ts @@ -10739,6 +10739,7 @@ export const data = [ integrations: { All: true, Amplitude: { + skipUserPropertiesSync: false, event_id: 2, }, }, @@ -10894,6 +10895,7 @@ export const data = [ integrations: { All: true, Amplitude: { + skipUserPropertiesSync: true, event_id: 2, }, }, @@ -10949,6 +10951,7 @@ export const data = [ insert_id: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', ip: '1.1.1.1', event_id: 2, + $skip_user_properties_sync: true, user_properties: { initial_referrer: 'https://docs.rudderstack.com', initial_referring_domain: 'docs.rudderstack.com', From 2ad12399040cbdc15453c2bb84f3aa9ab87e5c1f Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Mon, 18 Mar 2024 09:57:03 +0530 Subject: [PATCH 2/8] chore: onboard adobe to proxy v1 tests (#3163) * chore: onboard adobe to proxy v1 tests * chore: fix lint * chore: fix lintx2 --- .../adobe_analytics/dataDelivery/business.ts | 181 ++++++++++++++++++ .../adobe_analytics/dataDelivery/data.ts | 8 +- .../destinations/adobe_analytics/network.ts | 2 +- 3 files changed, 188 insertions(+), 3 deletions(-) create mode 100644 test/integrations/destinations/adobe_analytics/dataDelivery/business.ts diff --git a/test/integrations/destinations/adobe_analytics/dataDelivery/business.ts b/test/integrations/destinations/adobe_analytics/dataDelivery/business.ts new file mode 100644 index 0000000000..76e07690cf --- /dev/null +++ b/test/integrations/destinations/adobe_analytics/dataDelivery/business.ts @@ -0,0 +1,181 @@ +import { ProxyMetdata } from '../../../../../src/types'; +import { ProxyV1TestData } from '../../../testTypes'; +import { generateProxyV1Payload } from '../../../testUtils'; + +const statTags = { + aborted: { + destType: 'ADOBE_ANALYTICS', + destinationId: 'dummyDestinationId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'dummyWorkspaceId', + }, +}; + +export const proxyMetdata: ProxyMetdata = { + jobId: 1, + attemptNum: 1, + userId: 'dummyUserId', + sourceId: 'dummySourceId', + destinationId: 'dummyDestinationId', + workspaceId: 'dummyWorkspaceId', + secret: {}, + dontBatch: false, +}; +const headers = { + 'Content-Type': 'application/xml', +}; + +export const reqMetadataArray = [proxyMetdata]; + +const failureRequestParameters = { + XML: { + payload: + '17941080sales campaignwebUSD127.0.0.1en-USDalvik/2.1.0 (Linux; U; Android 9; Android SDK built for x86 Build/PSR1.180720.075)https://www.google.com/search?q=estore+bestsellerprodViewGames;;11;148.39failureReport', + }, + params: {}, +}; + +const successRequestParameters = { + XML: { + payload: + '127.0.1.0www.google.co.inGoogleid1110011prodViewGames;Monopoly;1;14.00,Games;UNO;2;6.90successreport', + }, + params: {}, +}; + +export const testScenariosForV1API: ProxyV1TestData[] = [ + { + id: 'adobe_analytics_v1_scenario_1', + name: 'adobe_analytics', + description: '[Proxy v1 API] :: Test for Failure response from Adobe Analytics with reason', + successCriteria: 'Should return a 400 status code with reason', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + ...failureRequestParameters, + headers, + endpoint: 'https://adobe.failure.omtrdc.net/b/ss//6', + }, + reqMetadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 400, + statTags: statTags.aborted, + message: + '[ADOBE_ANALYTICS Response Handler] Request failed for destination adobe_analytics : NO pagename OR pageurl', + response: [ + { + error: + '[ADOBE_ANALYTICS Response Handler] Request failed for destination adobe_analytics : NO pagename OR pageurl', + metadata: proxyMetdata, + statusCode: 400, + }, + ], + }, + }, + }, + }, + }, + { + id: 'adobe_analytics_v1_scenario_2', + name: 'adobe_analytics', + description: + '[Proxy v1 API] :: Test for Failure response from Adobe Analytics without reason (Generic error)', + successCriteria: 'Should return a 400 status code with a general error', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + ...failureRequestParameters, + headers, + endpoint: 'https://adobe.failure2.omtrdc.net/b/ss//6', + }, + reqMetadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 400, + statTags: statTags.aborted, + message: + '[ADOBE_ANALYTICS Response Handler] Request failed for destination adobe_analytics with a general error', + response: [ + { + error: + '[ADOBE_ANALYTICS Response Handler] Request failed for destination adobe_analytics with a general error', + metadata: proxyMetdata, + statusCode: 400, + }, + ], + }, + }, + }, + }, + }, + { + id: 'adobe_analytics_v1_scenario_3', + name: 'adobe_analytics', + description: '[Proxy v1 API] :: Test for Success response from Adobe Analytics', + successCriteria: 'Should return a 200 status code with status SUCCESS', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + ...successRequestParameters, + headers, + endpoint: 'https://adobe.success.omtrdc.net/b/ss//6', + }, + reqMetadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[ADOBE_ANALYTICS] - Request Processed Successfully', + response: [ + { + error: '"SUCCESS"', + metadata: proxyMetdata, + statusCode: 200, + }, + ], + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/adobe_analytics/dataDelivery/data.ts b/test/integrations/destinations/adobe_analytics/dataDelivery/data.ts index 182969da73..2535a0639e 100644 --- a/test/integrations/destinations/adobe_analytics/dataDelivery/data.ts +++ b/test/integrations/destinations/adobe_analytics/dataDelivery/data.ts @@ -1,4 +1,6 @@ -export const data = [ +import { testScenariosForV1API } from './business'; + +const legacyTests = [ { name: 'adobe_analytics', description: 'Test 0: Failure response from Adobe Analytics with reason', @@ -72,7 +74,7 @@ export const data = [ JSON_ARRAY: {}, XML: { payload: - '17941080sales campaignwebUSD127.0.0.1en-USDalvik/2.1.0 (Linux; U; Android 9; Android SDK built for x86 Build/PSR1.180720.075)https://www.google.com/search?q=estore+bestsellerprodViewGames;;11;148.39failureReportgeneric', + '17941080sales campaignwebUSD127.0.0.1en-USDalvik/2.1.0 (Linux; U; Android 9; Android SDK built for x86 Build/PSR1.180720.075)https://www.google.com/search?q=estore+bestsellerprodViewGames;;11;148.39failureReport', }, FORM: {}, }, @@ -140,3 +142,5 @@ export const data = [ }, }, ]; + +export const data = [...testScenariosForV1API, ...legacyTests]; diff --git a/test/integrations/destinations/adobe_analytics/network.ts b/test/integrations/destinations/adobe_analytics/network.ts index 2fe4f0204e..7e32c5f10b 100644 --- a/test/integrations/destinations/adobe_analytics/network.ts +++ b/test/integrations/destinations/adobe_analytics/network.ts @@ -17,7 +17,7 @@ export const networkCallsData = [ { httpReq: { url: 'https://adobe.failure2.omtrdc.net/b/ss//6', - data: '17941080sales campaignwebUSD127.0.0.1en-USDalvik/2.1.0 (Linux; U; Android 9; Android SDK built for x86 Build/PSR1.180720.075)https://www.google.com/search?q=estore+bestsellerprodViewGames;;11;148.39failureReportgeneric', + data: '17941080sales campaignwebUSD127.0.0.1en-USDalvik/2.1.0 (Linux; U; Android 9; Android SDK built for x86 Build/PSR1.180720.075)https://www.google.com/search?q=estore+bestsellerprodViewGames;;11;148.39failureReport', params: {}, headers: { 'Content-Type': 'application/xml', From 7018b1e5e7f37ae177191c5ecf3a71cfe2f3d147 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:36:32 +0530 Subject: [PATCH 3/8] feat: onboard destination movable ink (#3167) * feat: onboard destination movable ink * test: update common.ts * feat: add batching support * feat: batching on max size in bytes * docs: added comments --- src/cdk/v2/destinations/movable_ink/config.js | 3 + .../movable_ink/procWorkflow.yaml | 72 +++++++ .../destinations/movable_ink/rtWorkflow.yaml | 74 +++++++ src/features.json | 1 + .../destinations/movable_ink/common.ts | 128 ++++++++++++ .../movable_ink/processor/data.ts | 4 + .../movable_ink/processor/identify.ts | 64 ++++++ .../movable_ink/processor/track.ts | 189 ++++++++++++++++++ .../movable_ink/processor/validation.ts | 131 ++++++++++++ .../destinations/movable_ink/router/data.ts | 162 +++++++++++++++ 10 files changed, 828 insertions(+) create mode 100644 src/cdk/v2/destinations/movable_ink/config.js create mode 100644 src/cdk/v2/destinations/movable_ink/procWorkflow.yaml create mode 100644 src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml create mode 100644 test/integrations/destinations/movable_ink/common.ts create mode 100644 test/integrations/destinations/movable_ink/processor/data.ts create mode 100644 test/integrations/destinations/movable_ink/processor/identify.ts create mode 100644 test/integrations/destinations/movable_ink/processor/track.ts create mode 100644 test/integrations/destinations/movable_ink/processor/validation.ts create mode 100644 test/integrations/destinations/movable_ink/router/data.ts diff --git a/src/cdk/v2/destinations/movable_ink/config.js b/src/cdk/v2/destinations/movable_ink/config.js new file mode 100644 index 0000000000..673e94620e --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/config.js @@ -0,0 +1,3 @@ +module.exports = { + MAX_REQUEST_SIZE_IN_BYTES: 13500, +}; diff --git a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml new file mode 100644 index 0000000000..25270058c5 --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml @@ -0,0 +1,72 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: defaultRequestConfig + path: ../../../../v0/util + - name: toUnixTimestampInMS + path: ../../../../v0/util + - name: base64Convertor + path: ../../../../v0/util + - path: ./utils + +steps: + - name: messageType + template: | + .message.type.toLowerCase(); + + - name: validateInput + template: | + let messageType = $.outputs.messageType; + $.assert(messageType, "message Type is not present. Aborting"); + $.assert(messageType in {{$.EventType.([.IDENTIFY,.TRACK])}}, "message type " + messageType + " is not supported"); + $.assertConfig(.destination.Config.endpoint, "Movable Ink Endpoint is not present. Aborting"); + $.assertConfig(.destination.Config.accessKey, "Access key is not present . Aborting"); + $.assertConfig(.destination.Config.accessSecret, "Access Secret is not present. Aborting"); + $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting"); + + const userId = .message.().( + {{{{$.getGenericPaths("userIdOnly")}}}}; + ); + const email = .message.().( + {{{{$.getGenericPaths("email")}}}}; + ); + + $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting"); + + - name: preparePayload + description: Prepare payload for identify and track. This payload schema needs to be configured in the Movable Ink dashboard. Movable Ink will discard any additional fields from the input payload. + template: | + const userId = .message.().( + {{{{$.getGenericPaths("userIdOnly")}}}}; + ); + const email = .message.().( + {{{{$.getGenericPaths("email")}}}}; + ); + const timestampInUnix = $.toUnixTimestampInMS(.message.().( + {{{{$.getGenericPaths("timestamp")}}}}; + )); + $.context.payload = { + ...(.message), + userId: userId ?? email, + timestamp: timestampInUnix, + anonymousId: .message.anonymousId + } + + - name: buildResponse + description: In batchMode we return payload directly + condition: $.batchMode + template: | + $.context.payload + else: + name: buildResponseForProcessTransformation + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = $.context.payload; + response.endpoint = .destination.Config.endpoint; + response.method = "POST"; + response.headers = { + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.destination.Config.accessKey + ":" + .destination.Config.accessSecret) + } + response; diff --git a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml new file mode 100644 index 0000000000..46afb34d53 --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml @@ -0,0 +1,74 @@ +bindings: + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - path: ./utils + exportAll: true + - name: base64Convertor + path: ../../../../v0/util + - name: BatchUtils + path: '@rudderstack/workflow-engine' + - path: ./config + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + bindings: + - name: batchMode + value: true + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "batchedRequest": ., + "batched": false, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata, + "statusCode": 200 + })[] + + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + let batches = $.BatchUtils.chunkArrayBySizeAndLength( + $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES}).items; + + batches@batch.({ + "batchedRequest": { + "body": { + "JSON": {"events": ~r batch.batchedRequest[]}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": batch[0].destination.Config.().(.endpoint), + "headers": batch[0].destination.Config.().({ + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.accessKey + ":" + .accessSecret) + }), + "params": {}, + "files": {} + }, + "metadata": ~r batch.metadata[], + "batched": true, + "statusCode": 200, + "destination": batch[0].destination + })[]; + + - name: finalPayload + template: | + [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents] diff --git a/src/features.json b/src/features.json index dc52044048..76b562a825 100644 --- a/src/features.json +++ b/src/features.json @@ -67,6 +67,7 @@ "THE_TRADE_DESK": true, "INTERCOM": true, "NINETAILED": true, + "MOVABLE_INK": true, "KOALA": true }, "regulations": [ diff --git a/test/integrations/destinations/movable_ink/common.ts b/test/integrations/destinations/movable_ink/common.ts new file mode 100644 index 0000000000..f7eaa7af39 --- /dev/null +++ b/test/integrations/destinations/movable_ink/common.ts @@ -0,0 +1,128 @@ +import { Destination } from '../../../../src/types'; + +const destType = 'movable_ink'; +const destTypeInUpperCase = 'MOVABLE_INK'; +const displayName = 'Movable Ink'; +const channel = 'web'; +const destination: Destination = { + Config: { + endpoint: 'https://collector.movableink-dmz.com/behavioral/abc123', + accessKey: 'test-access-key', + accessSecret: 'test_access_secret', + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', +}; + +const processorInstrumentationErrorStatTags = { + destType: destTypeInUpperCase, + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', +}; + +const RouterInstrumentationErrorStatTags = { + ...processorInstrumentationErrorStatTags, + feature: 'router', +}; + +const traits = { + email: 'test@example.com', + firstName: 'John', + lastName: 'Doe', + phone: '1234567890', +}; + +const headers = { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC1hY2Nlc3Mta2V5OnRlc3RfYWNjZXNzX3NlY3JldA==', +}; + +const commonProperties = { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + categories: [ + { url: 'https://example1', id: '1' }, + { url: 'https://example2', id: '2' }, + ], + name: 'Cones of Dunshire', + brand: 'Wyatt Games', + variant: 'expansion pack', + price: 49.99, + quantity: 5, + coupon: 'PREORDER15', + position: 1, + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.webp', +}; + +const customProperties = { + key1: 'value1', + key2: true, + key3: ['value3'], + key4: { key5: { key6: 'value6' } }, +}; + +const trackTestProperties = { + 'Product Added': { ...commonProperties, ...customProperties }, + 'Product Viewed': { ...commonProperties, ...customProperties }, + 'Order Completed': { + checkout_id: '70324a1f0eaf000000000000', + order_id: '40684e8f0eaf000000000000', + affiliation: 'Vandelay Games', + total: 52, + subtotal: 45, + revenue: 50, + shipping: 4, + tax: 3, + discount: 5, + coupon: 'NEWCUST5', + currency: 'USD', + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + name: 'Cones of Dunshire', + price: 40, + position: 1, + category: 'Games', + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.jpg', + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + name: 'Five Crowns', + price: 5, + position: 2, + category: 'Games', + }, + ], + }, + 'Products Searched': { query: 'HDMI cable', url: 'https://www.website.com/product/path' }, + 'Custom event': { ...commonProperties, key1: 'value1', key2: true }, +}; + +export { + destType, + channel, + destination, + processorInstrumentationErrorStatTags, + RouterInstrumentationErrorStatTags, + traits, + headers, + trackTestProperties, +}; diff --git a/test/integrations/destinations/movable_ink/processor/data.ts b/test/integrations/destinations/movable_ink/processor/data.ts new file mode 100644 index 0000000000..45453c74cd --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/data.ts @@ -0,0 +1,4 @@ +import { validation } from './validation'; +import { identify } from './identify'; +import { track } from './track'; +export const data = [...identify, ...track, ...validation]; diff --git a/test/integrations/destinations/movable_ink/processor/identify.ts b/test/integrations/destinations/movable_ink/processor/identify.ts new file mode 100644 index 0000000000..27186da05c --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/identify.ts @@ -0,0 +1,64 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, channel, destination, traits, headers } from '../common'; + +export const identify: ProcessorTestData[] = [ + { + id: 'MovableInk-identify-test-1', + name: destType, + description: 'Identify call with traits and anonymousId', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'identify', + userId: traits.email, + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/processor/track.ts b/test/integrations/destinations/movable_ink/processor/track.ts new file mode 100644 index 0000000000..5f30a3de83 --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/track.ts @@ -0,0 +1,189 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, channel, destination, headers, trackTestProperties } from '../common'; + +export const track: ProcessorTestData[] = [ + { + id: 'MovableInk-track-test-1', + name: destType, + description: 'Track call: Product Added event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'MovableInk-track-test-2', + name: destType, + description: 'Track call: Order Completed event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'MovableInk-track-test-3', + name: destType, + description: 'Track call: Custom event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/processor/validation.ts b/test/integrations/destinations/movable_ink/processor/validation.ts new file mode 100644 index 0000000000..f9f6c6a927 --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/validation.ts @@ -0,0 +1,131 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata } from '../../../testUtils'; +import { destType, destination, processorInstrumentationErrorStatTags } from '../common'; + +export const validation: ProcessorTestData[] = [ + { + id: 'MovableInk-validation-test-1', + name: destType, + description: 'All of the required fields — userId, email, and anonymousId — are missing.', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Either one of userId or email or anonymousId is required. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Either one of userId or email or anonymousId is required. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'MovableInk-validation-test-2', + name: destType, + description: 'Unsupported message type -> group', + scenario: 'Framework', + successCriteria: 'Instrumentation Error for Unsupported message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'group', + userId: 'userId123', + channel: 'mobile', + anonymousId: 'anon_123', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'MovableInk-validation-test-3', + name: destType, + description: 'Missing required field -> timestamp', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Timestamp is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Timestamp is not present. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/router/data.ts b/test/integrations/destinations/movable_ink/router/data.ts new file mode 100644 index 0000000000..72df3d7074 --- /dev/null +++ b/test/integrations/destinations/movable_ink/router/data.ts @@ -0,0 +1,162 @@ +import { RouterTestData } from '../../../testTypes'; +import { RouterTransformationRequest } from '../../../../../src/types'; +import { generateMetadata } from '../../../testUtils'; +import { + destType, + channel, + destination, + traits, + headers, + trackTestProperties, + RouterInstrumentationErrorStatTags, +} from '../common'; + +const routerRequest: RouterTransformationRequest = { + input: [ + { + message: { + type: 'identify', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(2), + destination, + }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(3), + destination, + }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + }, + metadata: generateMetadata(4), + destination, + }, + ], + destType, +}; + +export const data: RouterTestData[] = [ + { + id: 'MovableInk-router-test-1', + name: destType, + description: 'Basic Router Test to test multiple payloads', + scenario: 'Framework', + successCriteria: + 'Some events should be transformed successfully and some should fail for missing fields and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + events: [ + { + type: 'identify', + userId: traits.email, + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1), generateMetadata(3)], + batched: true, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(2)], + batched: false, + statusCode: 400, + error: 'Either one of userId or email or anonymousId is required. Aborting', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(4)], + batched: false, + statusCode: 400, + error: 'Timestamp is not present. Aborting', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + ], + }, + }, + }, + }, +]; From c5105302f5d8f518b7caaf3e8e1012ca59cddde5 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 18 Mar 2024 07:22:12 +0000 Subject: [PATCH 4/8] chore(release): 1.59.0 --- CHANGELOG.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c143851091..7390317ab5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,57 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.59.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.59.0) (2024-03-18) + + +### Features + +* add Koala destination ([#3122](https://github.com/rudderlabs/rudder-transformer/issues/3122)) ([1ca039d](https://github.com/rudderlabs/rudder-transformer/commit/1ca039d64ebb1a18a0fc6b78ed5ee08528ad6b48)) +* add support for interaction events in sfmc ([#3109](https://github.com/rudderlabs/rudder-transformer/issues/3109)) ([0486049](https://github.com/rudderlabs/rudder-transformer/commit/0486049ba2ad96b50d8f29e96b46b96a8a5c9f76)) +* add support of ([c0ad214](https://github.com/rudderlabs/rudder-transformer/commit/c0ad21463981ef66154c8157083924f76825762d)) +* add support of custom page/screen event name in mixpanel ([#3098](https://github.com/rudderlabs/rudder-transformer/issues/3098)) ([0eb2393](https://github.com/rudderlabs/rudder-transformer/commit/0eb2393939fba2452ef7f07a1d149d87f18290c3)) +* add support of skip_user_properties_sync on Amplitude ([#3181](https://github.com/rudderlabs/rudder-transformer/issues/3181)) ([5e4ddbd](https://github.com/rudderlabs/rudder-transformer/commit/5e4ddbd8a591341a581a5721505d6dcb010f2eec)) +* adding zod validations ([#3066](https://github.com/rudderlabs/rudder-transformer/issues/3066)) ([325433b](https://github.com/rudderlabs/rudder-transformer/commit/325433b9188c8d1dbe740c7e193cdc2e58fdd751)) +* consent mode support for google adwords remarketing list ([#3143](https://github.com/rudderlabs/rudder-transformer/issues/3143)) ([7532c90](https://github.com/rudderlabs/rudder-transformer/commit/7532c90d7e1feac00f12961c56da18757010f44a)) +* **facebook:** update content_type mapping logic for fb pixel and fb conversions ([#3113](https://github.com/rudderlabs/rudder-transformer/issues/3113)) ([aea417c](https://github.com/rudderlabs/rudder-transformer/commit/aea417cd2691547399010c034cadbc5db6b0c6ee)) +* klaviyo profile mapping ([#3105](https://github.com/rudderlabs/rudder-transformer/issues/3105)) ([2761786](https://github.com/rudderlabs/rudder-transformer/commit/2761786ff3fc99ed6d4d3b7a6c2400226b1cfb12)) +* onboard destination movable ink ([#3167](https://github.com/rudderlabs/rudder-transformer/issues/3167)) ([7018b1e](https://github.com/rudderlabs/rudder-transformer/commit/7018b1e5e7f37ae177191c5ecf3a71cfe2f3d147)) +* onboard new destination ninetailed ([#3106](https://github.com/rudderlabs/rudder-transformer/issues/3106)) ([0e2588e](https://github.com/rudderlabs/rudder-transformer/commit/0e2588ecd87f3b2c6877a099aa1cbf2d5325966c)) +* update proxy data type for response handler input ([7d6ea12](https://github.com/rudderlabs/rudder-transformer/commit/7d6ea123e08b793a87f35290e740cbef547c3862)) +* update proxy tests for cm360 ([9dd8625](https://github.com/rudderlabs/rudder-transformer/commit/9dd862540cc8e4e56b9bc638cc1da62e5f19c45f)) +* update proxy tests for cm360 ([#3039](https://github.com/rudderlabs/rudder-transformer/issues/3039)) ([0504ffa](https://github.com/rudderlabs/rudder-transformer/commit/0504ffa898956f5b61771fb32ecfd0e0bf15248f)) +* update proxy v1 test cases ([b1327eb](https://github.com/rudderlabs/rudder-transformer/commit/b1327ebdb049163b3c5f046cb4605518e99481f3)) +* use dontBatch directive in algolia ([#3169](https://github.com/rudderlabs/rudder-transformer/issues/3169)) ([916aaec](https://github.com/rudderlabs/rudder-transformer/commit/916aaecb1939160620d5fd3c4c0c0e33f2a371b2)) + + +### Bug Fixes + +* add error handling for tiktok ads ([#3144](https://github.com/rudderlabs/rudder-transformer/issues/3144)) ([e93e47f](https://github.com/rudderlabs/rudder-transformer/commit/e93e47f33e098104fb532916932fe38bbfeaa4a1)) +* **algolia:** added check for objectIds or filters to be non empty ([#3126](https://github.com/rudderlabs/rudder-transformer/issues/3126)) ([d619c97](https://github.com/rudderlabs/rudder-transformer/commit/d619c9769cd270cb2d16dad0865683ff4beb2d19)) +* am formatting issues ([4653b74](https://github.com/rudderlabs/rudder-transformer/commit/4653b74522cc917230c211ce1df1b57e8a607ad7)) +* api contract for v1 proxy ([76e0284](https://github.com/rudderlabs/rudder-transformer/commit/76e02848c58a6630c36f724dc4ccbac3d29a8007)) +* api contract for v1 proxy ([#3049](https://github.com/rudderlabs/rudder-transformer/issues/3049)) ([93947db](https://github.com/rudderlabs/rudder-transformer/commit/93947db35cdaf1ca7ed87ec5f73567754af312ab)) +* clevertap remove stringification of array object properties ([#3048](https://github.com/rudderlabs/rudder-transformer/issues/3048)) ([69e43b6](https://github.com/rudderlabs/rudder-transformer/commit/69e43b6ffadeaec87b7440da34a341890ceba252)) +* convert to string from null in hs ([#3136](https://github.com/rudderlabs/rudder-transformer/issues/3136)) ([75e9f46](https://github.com/rudderlabs/rudder-transformer/commit/75e9f462b0ff9b9a8abab3c78dc7d147926e9e5e)) +* email mapping for clevertap ([c1b3736](https://github.com/rudderlabs/rudder-transformer/commit/c1b3736ab60c9582bdf1c4b07a761976de0da16f)) +* email mapping for clevertap ([#3173](https://github.com/rudderlabs/rudder-transformer/issues/3173)) ([04eab92](https://github.com/rudderlabs/rudder-transformer/commit/04eab92e1c383f9e8cdd5c845530a42a0af2932a)) +* event fix and added utility ([#3142](https://github.com/rudderlabs/rudder-transformer/issues/3142)) ([9b705b7](https://github.com/rudderlabs/rudder-transformer/commit/9b705b71a9d3a595ea0fbf532602c3941b0a18db)) +* fb pixel test case refactor ([#3075](https://github.com/rudderlabs/rudder-transformer/issues/3075)) ([cff7d1c](https://github.com/rudderlabs/rudder-transformer/commit/cff7d1c4578087a37614c0ef4529058481873479)) +* fixed 500 status for algolia dontBatch ([#3178](https://github.com/rudderlabs/rudder-transformer/issues/3178)) ([6330888](https://github.com/rudderlabs/rudder-transformer/commit/6330888ad5c67e3a800037b56501fc08da09e4d1)) +* label not present in prometheus metrics ([#3176](https://github.com/rudderlabs/rudder-transformer/issues/3176)) ([01d460c](https://github.com/rudderlabs/rudder-transformer/commit/01d460c3edaf39b35c4686516c9e9140be46aa5e)) +* metadata structure correction ([#3119](https://github.com/rudderlabs/rudder-transformer/issues/3119)) ([8351b5c](https://github.com/rudderlabs/rudder-transformer/commit/8351b5cbbf81bbc14b2f884feaae4ad3ca59a39a)) +* one_signal: Encode external_id in endpoint ([#3140](https://github.com/rudderlabs/rudder-transformer/issues/3140)) ([8a20886](https://github.com/rudderlabs/rudder-transformer/commit/8a2088608d6da4b35bbb506db2fc3df1e4d41f3b)) +* prepare-for-staging-deploy.yml ([afb2f45](https://github.com/rudderlabs/rudder-transformer/commit/afb2f450ddee0522e802327dce68ac33a04c9639)) +* prepare-for-staging-deploy.yml ([05ffe82](https://github.com/rudderlabs/rudder-transformer/commit/05ffe820e5c5a3b346f39c268dd49fca47568461)) +* rakuten: sync property mapping sourcekeys to rudderstack standard spec ([#3129](https://github.com/rudderlabs/rudder-transformer/issues/3129)) ([2ebff95](https://github.com/rudderlabs/rudder-transformer/commit/2ebff956ff2aa74b008a8de832a31d8774d2d47e)) +* reddit revenue mapping for floating point values ([#3118](https://github.com/rudderlabs/rudder-transformer/issues/3118)) ([41f4078](https://github.com/rudderlabs/rudder-transformer/commit/41f4078011ef54334bb9ecc11a7b2ccc8831a4aa)) +* release action git ([#3166](https://github.com/rudderlabs/rudder-transformer/issues/3166)) ([dff7eb9](https://github.com/rudderlabs/rudder-transformer/commit/dff7eb9b8072016a16e7083c60507a9d03302f17)) +* release fix feat, bug order ([#3165](https://github.com/rudderlabs/rudder-transformer/issues/3165)) ([17da0a9](https://github.com/rudderlabs/rudder-transformer/commit/17da0a9cd2efb7b3ae061db081c737cb38d30df2)) +* send proper status to server in cm360 ([#3127](https://github.com/rudderlabs/rudder-transformer/issues/3127)) ([229ce47](https://github.com/rudderlabs/rudder-transformer/commit/229ce473af1ddd62d946bea1b018c882b142a5ef)) +* typo ([650911e](https://github.com/rudderlabs/rudder-transformer/commit/650911e44c5c99f346f4bcfd8145fcd6993d7759)) +* upload js coverage to codecov ([#3179](https://github.com/rudderlabs/rudder-transformer/issues/3179)) ([d2eba21](https://github.com/rudderlabs/rudder-transformer/commit/d2eba21191dc4f7b610414158af68e5533016014)) +* version deprecation failure false positive ([#3104](https://github.com/rudderlabs/rudder-transformer/issues/3104)) ([657b780](https://github.com/rudderlabs/rudder-transformer/commit/657b7805eb01da25a007d978198d5debf03917fd)) + ## [1.58.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.58.0) (2024-03-04) diff --git a/package-lock.json b/package-lock.json index d708860537..63f19b12c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.58.0", + "version": "1.59.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.58.0", + "version": "1.59.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index ec3ffbf4e6..51b8b43a54 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.58.0", + "version": "1.59.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From ac7e8879fcd230dae09f757a759fb42e4cdc09d0 Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Mon, 18 Mar 2024 16:00:25 +0530 Subject: [PATCH 5/8] fix: update correct staging deployment file (#3189) --- .github/workflows/prepare-for-staging-deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/prepare-for-staging-deploy.yml b/.github/workflows/prepare-for-staging-deploy.yml index 1bd7e276f4..3e0b3aac19 100644 --- a/.github/workflows/prepare-for-staging-deploy.yml +++ b/.github/workflows/prepare-for-staging-deploy.yml @@ -112,9 +112,9 @@ jobs: yq eval -i ".user-transformer.image.tag=\"$TAG_NAME\"" staging.yaml git add staging.yaml - cd ../../../../config-be-rudder-transformer - yq eval -i ".config-be-rudder-transformer.image.tag=\"$TAG_NAME\"" values.staging.yaml - yq eval -i ".config-be-user-transformer.image.tag=\"$TAG_NAME\"" values.staging.yaml + cd ../../../../config-be-rudder-transformer/environment/staging + yq eval -i ".config-be-rudder-transformer.image.tag=\"$TAG_NAME\"" base.yaml + yq eval -i ".config-be-user-transformer.image.tag=\"$TAG_NAME\"" base.yaml git add values.staging.yaml git commit -m "chore: upgrade staging env transformers to \"$TAG_NAME\"" From 750fe8f73b153601e3c503b611d617236efb39a6 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Mon, 18 Mar 2024 18:18:58 +0530 Subject: [PATCH 6/8] chore: add event validation for movable ink destination (#3190) chore: add validation for movable ink destination --- .../movable_ink/procWorkflow.yaml | 1 + src/cdk/v2/destinations/movable_ink/utils.js | 21 +++++ .../movable_ink/processor/validation.ts | 86 +++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 src/cdk/v2/destinations/movable_ink/utils.js diff --git a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml index 25270058c5..43dbb3cbce 100644 --- a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml +++ b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml @@ -33,6 +33,7 @@ steps: ); $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting"); + $.validateEventPayload(.message); - name: preparePayload description: Prepare payload for identify and track. This payload schema needs to be configured in the Movable Ink dashboard. Movable Ink will discard any additional fields from the input payload. diff --git a/src/cdk/v2/destinations/movable_ink/utils.js b/src/cdk/v2/destinations/movable_ink/utils.js new file mode 100644 index 0000000000..04d7046b1a --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/utils.js @@ -0,0 +1,21 @@ +const { InstrumentationError } = require('@rudderstack/integrations-lib'); + +const validateEventPayload = (message) => { + const { event } = message; + const { properties } = message; + if (event === 'Products Searched' && !properties?.query) { + throw new InstrumentationError("Missing 'query' property in properties. Aborting"); + } + + if ( + (event === 'Product Added' || + event === 'Product Removed' || + event === 'Product Viewed' || + event === 'Category Viewed') && + !properties?.product_id + ) { + throw new InstrumentationError("Missing 'product_id' property in properties. Aborting"); + } +}; + +module.exports = { validateEventPayload }; diff --git a/test/integrations/destinations/movable_ink/processor/validation.ts b/test/integrations/destinations/movable_ink/processor/validation.ts index f9f6c6a927..ab6b123eb7 100644 --- a/test/integrations/destinations/movable_ink/processor/validation.ts +++ b/test/integrations/destinations/movable_ink/processor/validation.ts @@ -128,4 +128,90 @@ export const validation: ProcessorTestData[] = [ }, }, }, + { + id: 'MovableInk-validation-test-4', + name: destType, + description: "Products Searched event - Missing 'query' property", + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + userId: 'user123', + integrations: { + All: true, + }, + event: 'Products Searched', + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + "Missing 'query' property in properties. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Missing 'query' property in properties. Aborting", + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'MovableInk-validation-test-5', + name: destType, + description: "Products Added event - Missing 'product_id' property", + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + userId: 'user123', + integrations: { + All: true, + }, + event: 'Product Added', + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + "Missing 'product_id' property in properties. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Missing 'product_id' property in properties. Aborting", + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, ]; From 5876441004da398c635baca393c9acfb99794704 Mon Sep 17 00:00:00 2001 From: sandeepdigumarty Date: Tue, 19 Mar 2024 14:45:27 +0530 Subject: [PATCH 7/8] chore: updated CHANGELOG --- CHANGELOG.md | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7390317ab5..b624ed9ef7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,50 +8,27 @@ All notable changes to this project will be documented in this file. See [standa ### Features * add Koala destination ([#3122](https://github.com/rudderlabs/rudder-transformer/issues/3122)) ([1ca039d](https://github.com/rudderlabs/rudder-transformer/commit/1ca039d64ebb1a18a0fc6b78ed5ee08528ad6b48)) -* add support for interaction events in sfmc ([#3109](https://github.com/rudderlabs/rudder-transformer/issues/3109)) ([0486049](https://github.com/rudderlabs/rudder-transformer/commit/0486049ba2ad96b50d8f29e96b46b96a8a5c9f76)) -* add support of ([c0ad214](https://github.com/rudderlabs/rudder-transformer/commit/c0ad21463981ef66154c8157083924f76825762d)) -* add support of custom page/screen event name in mixpanel ([#3098](https://github.com/rudderlabs/rudder-transformer/issues/3098)) ([0eb2393](https://github.com/rudderlabs/rudder-transformer/commit/0eb2393939fba2452ef7f07a1d149d87f18290c3)) * add support of skip_user_properties_sync on Amplitude ([#3181](https://github.com/rudderlabs/rudder-transformer/issues/3181)) ([5e4ddbd](https://github.com/rudderlabs/rudder-transformer/commit/5e4ddbd8a591341a581a5721505d6dcb010f2eec)) * adding zod validations ([#3066](https://github.com/rudderlabs/rudder-transformer/issues/3066)) ([325433b](https://github.com/rudderlabs/rudder-transformer/commit/325433b9188c8d1dbe740c7e193cdc2e58fdd751)) -* consent mode support for google adwords remarketing list ([#3143](https://github.com/rudderlabs/rudder-transformer/issues/3143)) ([7532c90](https://github.com/rudderlabs/rudder-transformer/commit/7532c90d7e1feac00f12961c56da18757010f44a)) -* **facebook:** update content_type mapping logic for fb pixel and fb conversions ([#3113](https://github.com/rudderlabs/rudder-transformer/issues/3113)) ([aea417c](https://github.com/rudderlabs/rudder-transformer/commit/aea417cd2691547399010c034cadbc5db6b0c6ee)) -* klaviyo profile mapping ([#3105](https://github.com/rudderlabs/rudder-transformer/issues/3105)) ([2761786](https://github.com/rudderlabs/rudder-transformer/commit/2761786ff3fc99ed6d4d3b7a6c2400226b1cfb12)) * onboard destination movable ink ([#3167](https://github.com/rudderlabs/rudder-transformer/issues/3167)) ([7018b1e](https://github.com/rudderlabs/rudder-transformer/commit/7018b1e5e7f37ae177191c5ecf3a71cfe2f3d147)) -* onboard new destination ninetailed ([#3106](https://github.com/rudderlabs/rudder-transformer/issues/3106)) ([0e2588e](https://github.com/rudderlabs/rudder-transformer/commit/0e2588ecd87f3b2c6877a099aa1cbf2d5325966c)) -* update proxy data type for response handler input ([7d6ea12](https://github.com/rudderlabs/rudder-transformer/commit/7d6ea123e08b793a87f35290e740cbef547c3862)) -* update proxy tests for cm360 ([9dd8625](https://github.com/rudderlabs/rudder-transformer/commit/9dd862540cc8e4e56b9bc638cc1da62e5f19c45f)) * update proxy tests for cm360 ([#3039](https://github.com/rudderlabs/rudder-transformer/issues/3039)) ([0504ffa](https://github.com/rudderlabs/rudder-transformer/commit/0504ffa898956f5b61771fb32ecfd0e0bf15248f)) -* update proxy v1 test cases ([b1327eb](https://github.com/rudderlabs/rudder-transformer/commit/b1327ebdb049163b3c5f046cb4605518e99481f3)) * use dontBatch directive in algolia ([#3169](https://github.com/rudderlabs/rudder-transformer/issues/3169)) ([916aaec](https://github.com/rudderlabs/rudder-transformer/commit/916aaecb1939160620d5fd3c4c0c0e33f2a371b2)) ### Bug Fixes -* add error handling for tiktok ads ([#3144](https://github.com/rudderlabs/rudder-transformer/issues/3144)) ([e93e47f](https://github.com/rudderlabs/rudder-transformer/commit/e93e47f33e098104fb532916932fe38bbfeaa4a1)) -* **algolia:** added check for objectIds or filters to be non empty ([#3126](https://github.com/rudderlabs/rudder-transformer/issues/3126)) ([d619c97](https://github.com/rudderlabs/rudder-transformer/commit/d619c9769cd270cb2d16dad0865683ff4beb2d19)) * am formatting issues ([4653b74](https://github.com/rudderlabs/rudder-transformer/commit/4653b74522cc917230c211ce1df1b57e8a607ad7)) -* api contract for v1 proxy ([76e0284](https://github.com/rudderlabs/rudder-transformer/commit/76e02848c58a6630c36f724dc4ccbac3d29a8007)) * api contract for v1 proxy ([#3049](https://github.com/rudderlabs/rudder-transformer/issues/3049)) ([93947db](https://github.com/rudderlabs/rudder-transformer/commit/93947db35cdaf1ca7ed87ec5f73567754af312ab)) -* clevertap remove stringification of array object properties ([#3048](https://github.com/rudderlabs/rudder-transformer/issues/3048)) ([69e43b6](https://github.com/rudderlabs/rudder-transformer/commit/69e43b6ffadeaec87b7440da34a341890ceba252)) -* convert to string from null in hs ([#3136](https://github.com/rudderlabs/rudder-transformer/issues/3136)) ([75e9f46](https://github.com/rudderlabs/rudder-transformer/commit/75e9f462b0ff9b9a8abab3c78dc7d147926e9e5e)) -* email mapping for clevertap ([c1b3736](https://github.com/rudderlabs/rudder-transformer/commit/c1b3736ab60c9582bdf1c4b07a761976de0da16f)) * email mapping for clevertap ([#3173](https://github.com/rudderlabs/rudder-transformer/issues/3173)) ([04eab92](https://github.com/rudderlabs/rudder-transformer/commit/04eab92e1c383f9e8cdd5c845530a42a0af2932a)) -* event fix and added utility ([#3142](https://github.com/rudderlabs/rudder-transformer/issues/3142)) ([9b705b7](https://github.com/rudderlabs/rudder-transformer/commit/9b705b71a9d3a595ea0fbf532602c3941b0a18db)) * fb pixel test case refactor ([#3075](https://github.com/rudderlabs/rudder-transformer/issues/3075)) ([cff7d1c](https://github.com/rudderlabs/rudder-transformer/commit/cff7d1c4578087a37614c0ef4529058481873479)) * fixed 500 status for algolia dontBatch ([#3178](https://github.com/rudderlabs/rudder-transformer/issues/3178)) ([6330888](https://github.com/rudderlabs/rudder-transformer/commit/6330888ad5c67e3a800037b56501fc08da09e4d1)) * label not present in prometheus metrics ([#3176](https://github.com/rudderlabs/rudder-transformer/issues/3176)) ([01d460c](https://github.com/rudderlabs/rudder-transformer/commit/01d460c3edaf39b35c4686516c9e9140be46aa5e)) -* metadata structure correction ([#3119](https://github.com/rudderlabs/rudder-transformer/issues/3119)) ([8351b5c](https://github.com/rudderlabs/rudder-transformer/commit/8351b5cbbf81bbc14b2f884feaae4ad3ca59a39a)) -* one_signal: Encode external_id in endpoint ([#3140](https://github.com/rudderlabs/rudder-transformer/issues/3140)) ([8a20886](https://github.com/rudderlabs/rudder-transformer/commit/8a2088608d6da4b35bbb506db2fc3df1e4d41f3b)) * prepare-for-staging-deploy.yml ([afb2f45](https://github.com/rudderlabs/rudder-transformer/commit/afb2f450ddee0522e802327dce68ac33a04c9639)) * prepare-for-staging-deploy.yml ([05ffe82](https://github.com/rudderlabs/rudder-transformer/commit/05ffe820e5c5a3b346f39c268dd49fca47568461)) -* rakuten: sync property mapping sourcekeys to rudderstack standard spec ([#3129](https://github.com/rudderlabs/rudder-transformer/issues/3129)) ([2ebff95](https://github.com/rudderlabs/rudder-transformer/commit/2ebff956ff2aa74b008a8de832a31d8774d2d47e)) -* reddit revenue mapping for floating point values ([#3118](https://github.com/rudderlabs/rudder-transformer/issues/3118)) ([41f4078](https://github.com/rudderlabs/rudder-transformer/commit/41f4078011ef54334bb9ecc11a7b2ccc8831a4aa)) * release action git ([#3166](https://github.com/rudderlabs/rudder-transformer/issues/3166)) ([dff7eb9](https://github.com/rudderlabs/rudder-transformer/commit/dff7eb9b8072016a16e7083c60507a9d03302f17)) * release fix feat, bug order ([#3165](https://github.com/rudderlabs/rudder-transformer/issues/3165)) ([17da0a9](https://github.com/rudderlabs/rudder-transformer/commit/17da0a9cd2efb7b3ae061db081c737cb38d30df2)) * send proper status to server in cm360 ([#3127](https://github.com/rudderlabs/rudder-transformer/issues/3127)) ([229ce47](https://github.com/rudderlabs/rudder-transformer/commit/229ce473af1ddd62d946bea1b018c882b142a5ef)) -* typo ([650911e](https://github.com/rudderlabs/rudder-transformer/commit/650911e44c5c99f346f4bcfd8145fcd6993d7759)) * upload js coverage to codecov ([#3179](https://github.com/rudderlabs/rudder-transformer/issues/3179)) ([d2eba21](https://github.com/rudderlabs/rudder-transformer/commit/d2eba21191dc4f7b610414158af68e5533016014)) -* version deprecation failure false positive ([#3104](https://github.com/rudderlabs/rudder-transformer/issues/3104)) ([657b780](https://github.com/rudderlabs/rudder-transformer/commit/657b7805eb01da25a007d978198d5debf03917fd)) ## [1.58.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.58.0) (2024-03-04) From 05ebcc8269667cb2be9437d9779aec7328fe2bda Mon Sep 17 00:00:00 2001 From: sandeepdigumarty Date: Tue, 19 Mar 2024 15:42:38 +0530 Subject: [PATCH 8/8] chore: updated CHANGELOG --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b624ed9ef7..d3f2b57d19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,18 +17,12 @@ All notable changes to this project will be documented in this file. See [standa ### Bug Fixes -* am formatting issues ([4653b74](https://github.com/rudderlabs/rudder-transformer/commit/4653b74522cc917230c211ce1df1b57e8a607ad7)) * api contract for v1 proxy ([#3049](https://github.com/rudderlabs/rudder-transformer/issues/3049)) ([93947db](https://github.com/rudderlabs/rudder-transformer/commit/93947db35cdaf1ca7ed87ec5f73567754af312ab)) * email mapping for clevertap ([#3173](https://github.com/rudderlabs/rudder-transformer/issues/3173)) ([04eab92](https://github.com/rudderlabs/rudder-transformer/commit/04eab92e1c383f9e8cdd5c845530a42a0af2932a)) * fb pixel test case refactor ([#3075](https://github.com/rudderlabs/rudder-transformer/issues/3075)) ([cff7d1c](https://github.com/rudderlabs/rudder-transformer/commit/cff7d1c4578087a37614c0ef4529058481873479)) * fixed 500 status for algolia dontBatch ([#3178](https://github.com/rudderlabs/rudder-transformer/issues/3178)) ([6330888](https://github.com/rudderlabs/rudder-transformer/commit/6330888ad5c67e3a800037b56501fc08da09e4d1)) * label not present in prometheus metrics ([#3176](https://github.com/rudderlabs/rudder-transformer/issues/3176)) ([01d460c](https://github.com/rudderlabs/rudder-transformer/commit/01d460c3edaf39b35c4686516c9e9140be46aa5e)) -* prepare-for-staging-deploy.yml ([afb2f45](https://github.com/rudderlabs/rudder-transformer/commit/afb2f450ddee0522e802327dce68ac33a04c9639)) -* prepare-for-staging-deploy.yml ([05ffe82](https://github.com/rudderlabs/rudder-transformer/commit/05ffe820e5c5a3b346f39c268dd49fca47568461)) -* release action git ([#3166](https://github.com/rudderlabs/rudder-transformer/issues/3166)) ([dff7eb9](https://github.com/rudderlabs/rudder-transformer/commit/dff7eb9b8072016a16e7083c60507a9d03302f17)) -* release fix feat, bug order ([#3165](https://github.com/rudderlabs/rudder-transformer/issues/3165)) ([17da0a9](https://github.com/rudderlabs/rudder-transformer/commit/17da0a9cd2efb7b3ae061db081c737cb38d30df2)) * send proper status to server in cm360 ([#3127](https://github.com/rudderlabs/rudder-transformer/issues/3127)) ([229ce47](https://github.com/rudderlabs/rudder-transformer/commit/229ce473af1ddd62d946bea1b018c882b142a5ef)) -* upload js coverage to codecov ([#3179](https://github.com/rudderlabs/rudder-transformer/issues/3179)) ([d2eba21](https://github.com/rudderlabs/rudder-transformer/commit/d2eba21191dc4f7b610414158af68e5533016014)) ## [1.58.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.58.0) (2024-03-04)