From 8928def7d41bc849a8f31cfb1afd0f9d26cc0003 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 5 Feb 2024 09:22:22 +0000 Subject: [PATCH 01/13] chore(release): 1.55.0 --- CHANGELOG.md | 17 +++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ee4007dba..08032253ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.55.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.54.4...v1.55.0) (2024-02-05) + + +### Features + +* add new stat for access token expired in fb custom audience ([#3043](https://github.com/rudderlabs/rudder-transformer/issues/3043)) ([1e6d540](https://github.com/rudderlabs/rudder-transformer/commit/1e6d540fafc61a84fbbaa63d4bc5b1edc17ec44e)) +* **intercom:** upgrade intercom version from 1.4 to 2.10 ([#2976](https://github.com/rudderlabs/rudder-transformer/issues/2976)) ([717639b](https://github.com/rudderlabs/rudder-transformer/commit/717639bcce605109b145eb4cc6836fe1589278fe)) +* onboard new destination rakuten ([#3046](https://github.com/rudderlabs/rudder-transformer/issues/3046)) ([c7c3110](https://github.com/rudderlabs/rudder-transformer/commit/c7c3110a4526e31bc296abb33f3246fa8eee049a)) +* trade desk real time conversions ([#3023](https://github.com/rudderlabs/rudder-transformer/issues/3023)) ([212d5f0](https://github.com/rudderlabs/rudder-transformer/commit/212d5f09d8addc618d4398029e62c9a18a9512cf)) + + +### Bug Fixes + +* adding map for marketo known values and javascript known values ([#3037](https://github.com/rudderlabs/rudder-transformer/issues/3037)) ([64ab555](https://github.com/rudderlabs/rudder-transformer/commit/64ab555d31b4c1c49863794444bd79b2b6a45927)) +* mixpanel timestamp in ms ([#3028](https://github.com/rudderlabs/rudder-transformer/issues/3028)) ([5ad55a2](https://github.com/rudderlabs/rudder-transformer/commit/5ad55a27c8b737fd96f65c68ba086769747c5360)) +* upgrade ua-parser-js from 1.0.35 to 1.0.37 ([9a4cdef](https://github.com/rudderlabs/rudder-transformer/commit/9a4cdef59ab1c2d9dc95eb8629a7009d8d633297)) + ### [1.54.4](https://github.com/rudderlabs/rudder-transformer/compare/v1.54.3...v1.54.4) (2024-01-31) diff --git a/package-lock.json b/package-lock.json index 7fbf47bab5..dd42fd3921 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.54.4", + "version": "1.55.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.54.4", + "version": "1.55.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 13aaecb000..ac6746ed20 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.54.4", + "version": "1.55.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 61462b6646a921b4098c676883e0052901540104 Mon Sep 17 00:00:00 2001 From: Mihir Bhalala <77438541+mihir-4116@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:57:45 +0530 Subject: [PATCH 02/13] fix(intercom): retl edge case (#3058) --- .../destinations/intercom/procWorkflow.yaml | 2 +- .../destinations/intercom/processor/data.ts | 22 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/cdk/v2/destinations/intercom/procWorkflow.yaml b/src/cdk/v2/destinations/intercom/procWorkflow.yaml index 04afba9a25..4b2ca1869e 100644 --- a/src/cdk/v2/destinations/intercom/procWorkflow.yaml +++ b/src/cdk/v2/destinations/intercom/procWorkflow.yaml @@ -41,7 +41,7 @@ steps: version; - name: rEtlPayload - condition: .message.context.mappedToDestination === true + condition: .message.context.mappedToDestination template: | $.addExternalIdToTraits(.message); const payload = $.getFieldValueFromMessage(.message, "traits"); diff --git a/test/integrations/destinations/intercom/processor/data.ts b/test/integrations/destinations/intercom/processor/data.ts index 7ed9879b34..bd1b65b43e 100644 --- a/test/integrations/destinations/intercom/processor/data.ts +++ b/test/integrations/destinations/intercom/processor/data.ts @@ -1024,10 +1024,16 @@ export const data = [ body: [ { message: { - userId: 'user@1', channel: 'web', context: { - mappedToDestination: true, + externalId: [ + { + id: 'user@1', + type: 'INTERCOM-customer', + identifierType: 'user_id', + }, + ], + mappedToDestination: 'true', }, traits: { email: 'test@rudderlabs.com', @@ -1072,6 +1078,7 @@ export const data = [ name: 'Test Rudderlabs', phone: '+91 9999999999', owner_id: 13, + user_id: 'user@1', }, XML: {}, FORM: {}, @@ -1112,7 +1119,7 @@ export const data = [ userId: 'user@1', channel: 'web', context: { - mappedToDestination: true, + mappedToDestination: 'true', }, traits: { event_name: 'Product Viewed', @@ -2803,11 +2810,12 @@ export const data = [ }, externalId: [ { - identifierType: 'email', - id: 'test@gmail.com', + id: '10156', + type: 'INTERCOM-customer', + identifierType: 'user_id', }, ], - mappedToDestination: true, + mappedToDestination: 'true', device: { id: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', manufacturer: 'Apple', @@ -2905,8 +2913,8 @@ export const data = [ createdAt: '2020-09-30T19:11:00.337Z', phone: '9876543210', key1: 'value1', - email: 'test@gmail.com', update_last_request_at: true, + user_id: '10156', }, JSON_ARRAY: {}, XML: {}, From 6919a9e40f3a662548dd41e282d7565ff00525c3 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Tue, 6 Feb 2024 10:11:45 +0530 Subject: [PATCH 03/13] fix: trade desk enrichTrackPayload util (#3059) fix: trade desk extract custom properties util --- .../v2/destinations/the_trade_desk/utils.js | 9 ++++-- .../destinations/the_trade_desk/utils.test.js | 28 +++++++++++++------ .../the_trade_desk/router/data.ts | 14 ++++++++++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/cdk/v2/destinations/the_trade_desk/utils.js b/src/cdk/v2/destinations/the_trade_desk/utils.js index 64c5f2b78a..f51d8dc3ff 100644 --- a/src/cdk/v2/destinations/the_trade_desk/utils.js +++ b/src/cdk/v2/destinations/the_trade_desk/utils.js @@ -115,8 +115,10 @@ const prepareItemsPayload = (message) => { let items; const eventMapInfo = ECOMM_EVENT_MAP[event.toLowerCase()]; if (eventMapInfo?.itemsArray) { + // if event is one of the supported ecommerce events and products array is present items = prepareItemsFromProducts(message); } else if (eventMapInfo) { + // if event is one of the supported ecommerce events and products array is not present items = prepareItemsFromProperties(message); } return items; @@ -304,14 +306,17 @@ const enrichTrackPayload = (message, payload) => { if (eventsMapInfo && !eventsMapInfo.itemsArray) { const itemExclusionList = generateExclusionList(ITEM_CONFIGS); rawPayload = extractCustomFields(message, rawPayload, ['properties'], itemExclusionList); - } else { - // for custom events + } else if (eventsMapInfo) { + // for ecomm events with products array supports. e.g Order Completed event rawPayload = extractCustomFields( message, rawPayload, ['properties'], ['products', 'revenue', 'value'], ); + } else { + // for custom events + rawPayload = extractCustomFields(message, rawPayload, ['properties'], ['value']); } return rawPayload; }; diff --git a/src/cdk/v2/destinations/the_trade_desk/utils.test.js b/src/cdk/v2/destinations/the_trade_desk/utils.test.js index b489309956..029c3004ae 100644 --- a/src/cdk/v2/destinations/the_trade_desk/utils.test.js +++ b/src/cdk/v2/destinations/the_trade_desk/utils.test.js @@ -635,25 +635,35 @@ describe('enrichTrackPayload', () => { order_id: 'ord123', property1: 'value1', property2: 'value2', + revenue: 10, + value: 11, + products: [ + { + product_id: 'prd123', + test: 'test', + }, + ], }, }; const payload = { order_id: 'ord123', + value: 11, }; - let expectedPayload = { + const expectedPayload = { order_id: 'ord123', property1: 'value1', property2: 'value2', + revenue: 10, + value: 11, + products: [ + { + product_id: 'prd123', + test: 'test', + }, + ], }; - let result = enrichTrackPayload(message, payload); + const result = enrichTrackPayload(message, payload); expect(result).toEqual(expectedPayload); - - expectedPayload = { - order_id: 'ord123', - property1: 'value1', - property2: 'value2', - }; - expect(enrichTrackPayload(message, {})).toEqual(expectedPayload); }); }); diff --git a/test/integrations/destinations/the_trade_desk/router/data.ts b/test/integrations/destinations/the_trade_desk/router/data.ts index 6f379195fa..691ec703b9 100644 --- a/test/integrations/destinations/the_trade_desk/router/data.ts +++ b/test/integrations/destinations/the_trade_desk/router/data.ts @@ -1681,9 +1681,16 @@ export const data = [ properties: { key1: 'value1', value: 25, + revenue: 10, product_id: 'prd123', key2: true, test: 'test123', + products: [ + { + product_id: 'prd123', + test: 'test', + }, + ], }, }, destination: sampleDestination, @@ -1779,6 +1786,13 @@ export const data = [ test: 'test123', key1: 'value1', key2: true, + revenue: 10, + products: [ + { + product_id: 'prd123', + test: 'test', + }, + ], }, ], }, From 2a94bd845d1bdfcd16c553b89950b0a90ec30ec8 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 6 Feb 2024 10:26:23 +0530 Subject: [PATCH 04/13] chore: adding source parameter as rudderstack for rakuten --- src/cdk/v2/destinations/rakuten/procWorkflow.yaml | 2 +- test/integrations/destinations/rakuten/dataDelivery/data.ts | 1 + test/integrations/destinations/rakuten/network.ts | 4 ++++ test/integrations/destinations/rakuten/processor/track.ts | 4 ++++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/cdk/v2/destinations/rakuten/procWorkflow.yaml b/src/cdk/v2/destinations/rakuten/procWorkflow.yaml index 9ee9b5c03a..b4dcacfa09 100644 --- a/src/cdk/v2/destinations/rakuten/procWorkflow.yaml +++ b/src/cdk/v2/destinations/rakuten/procWorkflow.yaml @@ -23,7 +23,7 @@ steps: template: | const properties = $.constructProperties(.message); const lineItems = $.constructLineItems(.message.properties) - $.context.payload = {...properties,...lineItems,xml:1, mid:.destination.Config.mid} + $.context.payload = {...properties,...lineItems,xml:1,source:'rudderstack', mid:.destination.Config.mid} $.context.payload = $.removeUndefinedAndNullValues($.context.payload); - name: buildResponse diff --git a/test/integrations/destinations/rakuten/dataDelivery/data.ts b/test/integrations/destinations/rakuten/dataDelivery/data.ts index 2d2b00a5e4..ff40954fdf 100644 --- a/test/integrations/destinations/rakuten/dataDelivery/data.ts +++ b/test/integrations/destinations/rakuten/dataDelivery/data.ts @@ -1,6 +1,7 @@ import { endpoint, commonOutputHeaders } from '../processor/commonConfig'; const commonParams = { xml: 1, + source: 'rudderstack', amtlist: '12500|12500', qlist: '|5', ord: 'SampleOrderId', diff --git a/test/integrations/destinations/rakuten/network.ts b/test/integrations/destinations/rakuten/network.ts index 9633ee54a1..a9770d83e2 100644 --- a/test/integrations/destinations/rakuten/network.ts +++ b/test/integrations/destinations/rakuten/network.ts @@ -6,6 +6,7 @@ export const networkCallsData = [ params: { mid: 'invalid_mid', xml: 1, + source: 'rudderstack', amtlist: '12500|12500', qlist: '|5', ord: 'SampleOrderId', @@ -30,6 +31,7 @@ export const networkCallsData = [ params: { mid: 'access_denied_for_mid', xml: 1, + source: 'rudderstack', amtlist: '12500|12500', qlist: '|5', ord: 'SampleOrderId', @@ -54,6 +56,7 @@ export const networkCallsData = [ params: { mid: 'valid_mid_with_good_records', xml: 1, + source: 'rudderstack', amtlist: '12500|12500', qlist: '|5', ord: 'SampleOrderId', @@ -78,6 +81,7 @@ export const networkCallsData = [ params: { mid: 'valid_mid_with_bad_records', xml: 1, + source: 'rudderstack', amtlist: '12500|12500', qlist: '|5', ord: 'SampleOrderId', diff --git a/test/integrations/destinations/rakuten/processor/track.ts b/test/integrations/destinations/rakuten/processor/track.ts index 78a76e4263..49b26e4658 100644 --- a/test/integrations/destinations/rakuten/processor/track.ts +++ b/test/integrations/destinations/rakuten/processor/track.ts @@ -80,6 +80,7 @@ export const trackSuccess = [ params: { mid: 'dummyMarketingId', xml: 1, + source: 'rudderstack', amtlist: '2000|2500|3000', brandlist: 'SampleBrand||', catidlist: '12345||', @@ -209,6 +210,7 @@ export const trackSuccess = [ params: { mid: 'dummyMarketingId', xml: 1, + source: 'rudderstack', amtlist: '12500|12500|3000', couponlist: '||SALE50', namelist: 'name_1|name_2|SampleProduct', @@ -316,6 +318,7 @@ export const trackSuccess = [ params: { mid: 'dummyMarketingId', xml: 1, + source: 'rudderstack', amtlist: '-12500|-12500', skulist: 'custom sku 0|custom sku 1', qlist: '1|5', @@ -429,6 +432,7 @@ export const trackSuccess = [ params: { mid: 'dummyMarketingId', xml: 1, + source: 'rudderstack', namelist: 'name_1|name_2|Discount', amtlist: '12500|12500|-50000', skulist: 'custom sku 0|custom sku 1|Discount', From 7f04a5928b3981dc98d7afff4e27dc2d0f095f85 Mon Sep 17 00:00:00 2001 From: Mihir Bhalala <77438541+mihir-4116@users.noreply.github.com> Date: Tue, 6 Feb 2024 15:07:42 +0530 Subject: [PATCH 05/13] chore: restore intercom js code (#3062) --- src/v0/destinations/intercom/config.js | 53 +++ .../intercom/data/INTERCOMGroupConfig.json | 53 +++ .../intercom/data/INTERCOMIdentifyConfig.json | 46 ++ .../intercom/data/INTERCOMTrackConfig.json | 36 ++ src/v0/destinations/intercom/transform.js | 252 +++++++++++ src/v0/destinations/intercom/util.js | 32 ++ src/v0/destinations/intercom/util.test.js | 176 ++++++++ .../destinations/intercom/processor/data.ts | 403 ++++++++++++++++++ .../destinations/intercom/router/data.ts | 390 +++++++++++++++++ 9 files changed, 1441 insertions(+) create mode 100644 src/v0/destinations/intercom/config.js create mode 100644 src/v0/destinations/intercom/data/INTERCOMGroupConfig.json create mode 100644 src/v0/destinations/intercom/data/INTERCOMIdentifyConfig.json create mode 100644 src/v0/destinations/intercom/data/INTERCOMTrackConfig.json create mode 100644 src/v0/destinations/intercom/transform.js create mode 100644 src/v0/destinations/intercom/util.js create mode 100644 src/v0/destinations/intercom/util.test.js diff --git a/src/v0/destinations/intercom/config.js b/src/v0/destinations/intercom/config.js new file mode 100644 index 0000000000..ae29eebc1e --- /dev/null +++ b/src/v0/destinations/intercom/config.js @@ -0,0 +1,53 @@ +const { getMappingConfig } = require('../../util'); + +const BASE_ENDPOINT = 'https://api.intercom.io'; + +// track events | Track +const TRACK_ENDPOINT = `${BASE_ENDPOINT}/events`; +// Create, Update a user with a company | Identify +const IDENTIFY_ENDPOINT = `${BASE_ENDPOINT}/users`; +// create, update, delete a company | Group +const GROUP_ENDPOINT = `${BASE_ENDPOINT}/companies`; + +const ConfigCategory = { + TRACK: { + endpoint: TRACK_ENDPOINT, + name: 'INTERCOMTrackConfig', + }, + IDENTIFY: { + endpoint: IDENTIFY_ENDPOINT, + name: 'INTERCOMIdentifyConfig', + }, + GROUP: { + endpoint: GROUP_ENDPOINT, + name: 'INTERCOMGroupConfig', + }, +}; + +const MappingConfig = getMappingConfig(ConfigCategory, __dirname); + +const ReservedTraitsProperties = [ + 'userId', + 'email', + 'phone', + 'name', + 'createdAt', + 'firstName', + 'lastName', + 'firstname', + 'lastname', + 'company', +]; + +const ReservedCompanyProperties = ['id', 'name', 'industry']; + +// ref:- https://developers.intercom.com/intercom-api-reference/v1.4/reference/event-metadata-types +const MetadataTypes = { richLink: ['url', 'value'], monetaryAmount: ['amount', 'currency'] }; + +module.exports = { + ConfigCategory, + MappingConfig, + ReservedCompanyProperties, + ReservedTraitsProperties, + MetadataTypes, +}; diff --git a/src/v0/destinations/intercom/data/INTERCOMGroupConfig.json b/src/v0/destinations/intercom/data/INTERCOMGroupConfig.json new file mode 100644 index 0000000000..6857c4e104 --- /dev/null +++ b/src/v0/destinations/intercom/data/INTERCOMGroupConfig.json @@ -0,0 +1,53 @@ +[ + { + "destKey": "company_id", + "sourceKeys": "groupId", + "required": true + }, + { + "destKey": "name", + "sourceKeys": "name", + "sourceFromGenericMap": true, + "required": false + }, + { + "destKey": "plan", + "sourceKeys": ["traits.plan", "context.traits.plan"], + "required": false + }, + { + "destKey": "size", + "sourceKeys": ["traits.size", "context.traits.size"], + "metadata": { + "type": "toNumber" + }, + "required": false + }, + { + "destKey": "website", + "sourceKeys": "website", + "sourceFromGenericMap": true, + "required": false + }, + { + "destKey": "industry", + "sourceKeys": ["traits.industry", "context.traits.industry"], + "required": false + }, + { + "destKey": "monthly_spend", + "sourceKeys": ["traits.monthlySpend", "context.traits.monthlySpend"], + "metadata": { + "type": "toNumber" + }, + "required": false + }, + { + "destKey": "remote_created_at", + "sourceKeys": ["traits.remoteCreatedAt", "context.traits.remoteCreatedAt"], + "metadata": { + "type": "toNumber" + }, + "required": false + } +] diff --git a/src/v0/destinations/intercom/data/INTERCOMIdentifyConfig.json b/src/v0/destinations/intercom/data/INTERCOMIdentifyConfig.json new file mode 100644 index 0000000000..726a741161 --- /dev/null +++ b/src/v0/destinations/intercom/data/INTERCOMIdentifyConfig.json @@ -0,0 +1,46 @@ +[ + { + "destKey": "user_id", + "sourceKeys": [ + "userId", + "traits.userId", + "traits.id", + "context.traits.userId", + "context.traits.id" + ], + "required": false + }, + { + "destKey": "email", + "sourceKeys": ["traits.email", "context.traits.email"], + "required": false + }, + { + "destKey": "phone", + "sourceKeys": ["traits.phone", "context.traits.phone"], + "required": false + }, + { + "destKey": "name", + "sourceKeys": ["traits.name", "context.traits.name"], + "required": false + }, + { + "destKey": "signed_up_at", + "sourceKeys": ["traits.createdAt", "context.traits.createdAt"], + "required": false, + "metadata": { + "type": "secondTimestamp" + } + }, + { + "destKey": "last_seen_user_agent", + "sourceKeys": "context.userAgent", + "required": false + }, + { + "destKey": "custom_attributes", + "sourceKeys": ["traits", "context.traits"], + "required": false + } +] diff --git a/src/v0/destinations/intercom/data/INTERCOMTrackConfig.json b/src/v0/destinations/intercom/data/INTERCOMTrackConfig.json new file mode 100644 index 0000000000..f33c9a8a98 --- /dev/null +++ b/src/v0/destinations/intercom/data/INTERCOMTrackConfig.json @@ -0,0 +1,36 @@ +[ + { + "destKey": "user_id", + "sourceKeys": [ + "userId", + "traits.userId", + "traits.id", + "context.traits.userId", + "context.traits.id" + ], + "required": false + }, + { + "destKey": "email", + "sourceKeys": ["traits.email", "context.traits.email"], + "required": false + }, + { + "destKey": "event_name", + "sourceKeys": "event", + "required": true + }, + { + "destKey": "created", + "sourceKeys": "timestamp", + "sourceFromGenericMap": true, + "required": true, + "metadata": { + "type": "secondTimestamp" + } + }, + { + "destKey": "metadata", + "sourceKeys": "properties" + } +] diff --git a/src/v0/destinations/intercom/transform.js b/src/v0/destinations/intercom/transform.js new file mode 100644 index 0000000000..212eaba13b --- /dev/null +++ b/src/v0/destinations/intercom/transform.js @@ -0,0 +1,252 @@ +const md5 = require('md5'); +const get = require('get-value'); +const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { EventType, MappedToDestinationKey } = require('../../../constants'); +const { + ConfigCategory, + MappingConfig, + ReservedTraitsProperties, + ReservedCompanyProperties, +} = require('./config'); +const { + constructPayload, + removeUndefinedAndNullValues, + defaultRequestConfig, + defaultPostRequestConfig, + getFieldValueFromMessage, + addExternalIdToTraits, + simpleProcessRouterDest, + flattenJson, +} = require('../../util'); +const { separateReservedAndRestMetadata } = require('./util'); +const { JSON_MIME_TYPE } = require('../../util/constant'); + +function getCompanyAttribute(company) { + const companiesList = []; + if (company.name || company.id) { + const customAttributes = {}; + Object.keys(company).forEach((key) => { + // the key is not in ReservedCompanyProperties + if (!ReservedCompanyProperties.includes(key)) { + const val = company[key]; + if (val !== Object(val)) { + customAttributes[key] = val; + } else { + customAttributes[key] = JSON.stringify(val); + } + } + }); + + companiesList.push({ + company_id: company.id || md5(company.name), + custom_attributes: removeUndefinedAndNullValues(customAttributes), + name: company.name, + industry: company.industry, + }); + } + return companiesList; +} + +function validateIdentify(message, payload, config) { + const finalPayload = payload; + + finalPayload.update_last_request_at = + config.updateLastRequestAt !== undefined ? config.updateLastRequestAt : true; + if (payload.user_id || payload.email) { + if (payload.name === undefined || payload.name === '') { + const firstName = getFieldValueFromMessage(message, 'firstName'); + const lastName = getFieldValueFromMessage(message, 'lastName'); + if (firstName && lastName) { + finalPayload.name = `${firstName} ${lastName}`; + } else { + finalPayload.name = firstName || lastName; + } + } + + if (get(finalPayload, 'custom_attributes.company')) { + finalPayload.companies = getCompanyAttribute(finalPayload.custom_attributes.company); + } + + if (finalPayload.custom_attributes) { + ReservedTraitsProperties.forEach((trait) => { + delete finalPayload.custom_attributes[trait]; + }); + finalPayload.custom_attributes = flattenJson(finalPayload.custom_attributes); + } + + return finalPayload; + } + throw new InstrumentationError('Either of `email` or `userId` is required for Identify call'); +} + +function validateTrack(payload) { + if (!payload.user_id && !payload.email) { + throw new InstrumentationError('Either of `email` or `userId` is required for Track call'); + } + // pass only string, number, boolean properties + if (payload.metadata) { + // reserved metadata contains JSON objects that does not requires flattening + const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(payload.metadata); + return { ...payload, metadata: { ...reservedMetadata, ...flattenJson(restMetadata) } }; + } + + return payload; +} + +const checkIfEmailOrUserIdPresent = (message, Config) => { + const { context, anonymousId } = message; + let { userId } = message; + if (Config.sendAnonymousId && !userId) { + userId = anonymousId; + } + return !!(userId || context.traits?.email); +}; + +function attachUserAndCompany(message, Config) { + const email = message.context?.traits?.email; + const { userId, anonymousId, traits, groupId } = message; + const requestBody = {}; + if (userId) { + requestBody.user_id = userId; + } + if (Config.sendAnonymousId && !userId) { + requestBody.user_id = anonymousId; + } + if (email) { + requestBody.email = email; + } + const companyObj = { + company_id: groupId, + }; + if (traits?.name) { + companyObj.name = traits.name; + } + requestBody.companies = [companyObj]; + const response = defaultRequestConfig(); + response.method = defaultPostRequestConfig.requestMethod; + response.endpoint = ConfigCategory.IDENTIFY.endpoint; + response.headers = { + 'Content-Type': JSON_MIME_TYPE, + Authorization: `Bearer ${Config.apiKey}`, + Accept: JSON_MIME_TYPE, + 'Intercom-Version': '1.4', + }; + response.body.JSON = requestBody; + return response; +} + +function buildCustomAttributes(message, payload) { + const finalPayload = payload; + const { traits } = message; + const customAttributes = {}; + const companyReservedKeys = [ + 'remoteCreatedAt', + 'monthlySpend', + 'industry', + 'website', + 'size', + 'plan', + 'name', + ]; + + if (traits) { + Object.keys(traits).forEach((key) => { + if (!companyReservedKeys.includes(key) && key !== 'userId') { + customAttributes[key] = traits[key]; + } + }); + } + + if (Object.keys(customAttributes).length > 0) { + finalPayload.custom_attributes = flattenJson(customAttributes); + } + + return finalPayload; +} + +function validateAndBuildResponse(message, payload, category, destination) { + const respList = []; + const response = defaultRequestConfig(); + response.method = defaultPostRequestConfig.requestMethod; + response.endpoint = category.endpoint; + response.headers = { + 'Content-Type': JSON_MIME_TYPE, + Authorization: `Bearer ${destination.Config.apiKey}`, + Accept: JSON_MIME_TYPE, + 'Intercom-Version': '1.4', + }; + response.userId = message.anonymousId; + const messageType = message.type.toLowerCase(); + switch (messageType) { + case EventType.IDENTIFY: + response.body.JSON = removeUndefinedAndNullValues( + validateIdentify(message, payload, destination.Config), + ); + break; + case EventType.TRACK: + response.body.JSON = removeUndefinedAndNullValues(validateTrack(payload)); + break; + case EventType.GROUP: { + response.body.JSON = removeUndefinedAndNullValues(buildCustomAttributes(message, payload)); + respList.push(response); + if (checkIfEmailOrUserIdPresent(message, destination.Config)) { + const attachUserAndCompanyResponse = attachUserAndCompany(message, destination.Config); + attachUserAndCompanyResponse.userId = message.anonymousId; + respList.push(attachUserAndCompanyResponse); + } + break; + } + default: + throw new InstrumentationError(`Message type ${messageType} not supported`); + } + + return messageType === EventType.GROUP ? respList : response; +} + +function processSingleMessage(message, destination) { + if (!message.type) { + throw new InstrumentationError('Message Type is not present. Aborting message.'); + } + const { sendAnonymousId } = destination.Config; + const messageType = message.type.toLowerCase(); + let category; + + switch (messageType) { + case EventType.IDENTIFY: + category = ConfigCategory.IDENTIFY; + break; + case EventType.TRACK: + category = ConfigCategory.TRACK; + break; + case EventType.GROUP: + category = ConfigCategory.GROUP; + break; + default: + throw new InstrumentationError(`Message type ${messageType} not supported`); + } + + // build the response and return + let payload; + if (get(message, MappedToDestinationKey)) { + addExternalIdToTraits(message); + payload = getFieldValueFromMessage(message, 'traits'); + } else { + payload = constructPayload(message, MappingConfig[category.name]); + } + if (category !== ConfigCategory.GROUP && sendAnonymousId && !payload.user_id) { + payload.user_id = message.anonymousId; + } + return validateAndBuildResponse(message, payload, category, destination); +} + +function process(event) { + const response = processSingleMessage(event.message, event.destination); + return response; +} + +const processRouterDest = async (inputs, reqMetadata) => { + const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); + return respList; +}; + +module.exports = { process, processRouterDest }; diff --git a/src/v0/destinations/intercom/util.js b/src/v0/destinations/intercom/util.js new file mode 100644 index 0000000000..24a2934f7e --- /dev/null +++ b/src/v0/destinations/intercom/util.js @@ -0,0 +1,32 @@ +const { MetadataTypes } = require('./config'); + +/** + * Separates reserved metadata from rest of the metadata based on the metadata types + * ref:- https://developers.intercom.com/intercom-api-reference/v1.4/reference/event-metadata-types + * @param {*} metadata + * @returns + */ +function separateReservedAndRestMetadata(metadata) { + const reservedMetadata = {}; + const restMetadata = {}; + if (metadata) { + Object.entries(metadata).forEach(([key, value]) => { + if (value && typeof value === 'object') { + const hasMonetaryAmountKeys = MetadataTypes.monetaryAmount.every((type) => type in value); + const hasRichLinkKeys = MetadataTypes.richLink.every((type) => type in value); + if (hasMonetaryAmountKeys || hasRichLinkKeys) { + reservedMetadata[key] = value; + } else { + restMetadata[key] = value; + } + } else { + restMetadata[key] = value; + } + }); + } + + // Return the separated metadata objects + return { reservedMetadata, restMetadata }; +} + +module.exports = { separateReservedAndRestMetadata }; diff --git a/src/v0/destinations/intercom/util.test.js b/src/v0/destinations/intercom/util.test.js new file mode 100644 index 0000000000..99dbdd1f7e --- /dev/null +++ b/src/v0/destinations/intercom/util.test.js @@ -0,0 +1,176 @@ +const { separateReservedAndRestMetadata } = require('./util'); + +describe('separateReservedAndRestMetadata utility test', () => { + it('separate reserved and rest metadata', () => { + const metadata = { + property1: 1, + property2: 'test', + property3: true, + property4: { + property1: 1, + property2: 'test', + property3: { + subProp1: { + a: 'a', + b: 'b', + }, + subProp2: ['a', 'b'], + }, + }, + property5: {}, + property6: [], + property7: null, + property8: undefined, + revenue: { + amount: 1232, + currency: 'inr', + test: 123, + }, + price: { + amount: 3000, + currency: 'USD', + }, + article: { + url: 'https://example.org/ab1de.html', + value: 'the dude abides', + }, + }; + const expectedReservedMetadata = { + revenue: { + amount: 1232, + currency: 'inr', + test: 123, + }, + price: { + amount: 3000, + currency: 'USD', + }, + article: { + url: 'https://example.org/ab1de.html', + value: 'the dude abides', + }, + }; + const expectedRestMetadata = { + property1: 1, + property2: 'test', + property3: true, + property4: { + property1: 1, + property2: 'test', + property3: { + subProp1: { + a: 'a', + b: 'b', + }, + subProp2: ['a', 'b'], + }, + }, + property5: {}, + property6: [], + property7: null, + property8: undefined, + }; + const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(metadata); + + expect(expectedReservedMetadata).toEqual(reservedMetadata); + expect(expectedRestMetadata).toEqual(restMetadata); + }); + + it('reserved metadata types not present in input metadata', () => { + const metadata = { + property1: 1, + property2: 'test', + property3: true, + property4: { + property1: 1, + property2: 'test', + property3: { + subProp1: { + a: 'a', + b: 'b', + }, + subProp2: ['a', 'b'], + }, + }, + property5: {}, + property6: [], + property7: null, + property8: undefined, + }; + const expectedRestMetadata = { + property1: 1, + property2: 'test', + property3: true, + property4: { + property1: 1, + property2: 'test', + property3: { + subProp1: { + a: 'a', + b: 'b', + }, + subProp2: ['a', 'b'], + }, + }, + property5: {}, + property6: [], + property7: null, + property8: undefined, + }; + const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(metadata); + + expect({}).toEqual(reservedMetadata); + expect(expectedRestMetadata).toEqual(restMetadata); + }); + + it('metadata input contains only reserved metadata types', () => { + const metadata = { + revenue: { + amount: 1232, + currency: 'inr', + test: 123, + }, + price: { + amount: 3000, + currency: 'USD', + }, + article: { + url: 'https://example.org/ab1de.html', + value: 'the dude abides', + }, + }; + const expectedReservedMetadata = { + revenue: { + amount: 1232, + currency: 'inr', + test: 123, + }, + price: { + amount: 3000, + currency: 'USD', + }, + article: { + url: 'https://example.org/ab1de.html', + value: 'the dude abides', + }, + }; + const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(metadata); + + expect(expectedReservedMetadata).toEqual(reservedMetadata); + expect({}).toEqual(restMetadata); + }); + + it('empty metadata object', () => { + const metadata = {}; + const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(metadata); + expect({}).toEqual(reservedMetadata); + expect({}).toEqual(restMetadata); + }); + + it('null/undefined metadata', () => { + const metadata = null; + const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(metadata); + expect({}).toEqual(reservedMetadata); + expect({}).toEqual(restMetadata); + }); +}); diff --git a/test/integrations/destinations/intercom/processor/data.ts b/test/integrations/destinations/intercom/processor/data.ts index bd1b65b43e..2c562ed4e9 100644 --- a/test/integrations/destinations/intercom/processor/data.ts +++ b/test/integrations/destinations/intercom/processor/data.ts @@ -3743,4 +3743,407 @@ export const data = [ }, }, }, + { + name: 'intercom', + description: 'Test 0', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + channel: 'mobile', + context: { + app: { + build: '1.0', + name: 'Test_Example', + namespace: 'com.example.testapp', + version: '1.0', + }, + device: { + id: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + manufacturer: 'Apple', + model: 'iPhone', + name: 'iPod touch (7th generation)', + type: 'iOS', + }, + library: { + name: 'test-ios-library', + version: '1.0.7', + }, + locale: 'en-US', + network: { + bluetooth: false, + carrier: 'unavailable', + cellular: false, + wifi: true, + }, + os: { + name: 'iOS', + version: '14.0', + }, + screen: { + density: 2, + height: 320, + width: 568, + }, + timezone: 'Asia/Kolkata', + traits: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + name: 'Test Name', + firstName: 'Test', + lastName: 'Name', + createdAt: '2020-09-30T19:11:00.337Z', + userId: 'test_user_id_1', + email: 'test_1@test.com', + phone: '9876543210', + key1: 'value1', + address: { + city: 'Kolkata', + state: 'West Bengal', + }, + originalArray: [ + { + nested_field: 'nested value', + tags: ['tag_1', 'tag_2', 'tag_3'], + }, + { + nested_field: 'nested value', + tags: ['tag_1'], + }, + { + nested_field: 'nested value', + }, + ], + }, + userAgent: 'unknown', + }, + event: 'Test Event 2', + integrations: { + All: true, + }, + messageId: '1601493060-39010c49-e6e4-4626-a75c-0dbf1925c9e8', + originalTimestamp: '2020-09-30T19:11:00.337Z', + receivedAt: '2020-10-01T00:41:11.369+05:30', + request_ip: '2405:201:8005:9856:7911:25e7:5603:5e18', + sentAt: '2020-09-30T19:11:10.382Z', + timestamp: '2020-10-01T00:41:01.324+05:30', + type: 'identify', + }, + destination: { + Config: { + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.intercom.io/users', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer intercomApiKey', + Accept: 'application/json', + 'Intercom-Version': '1.4', + }, + params: {}, + body: { + JSON: { + user_id: 'test_user_id_1', + email: 'test_1@test.com', + phone: '9876543210', + name: 'Test Name', + signed_up_at: 1601493060, + last_seen_user_agent: 'unknown', + custom_attributes: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + key1: 'value1', + 'address.city': 'Kolkata', + 'address.state': 'West Bengal', + 'originalArray[0].nested_field': 'nested value', + 'originalArray[0].tags[0]': 'tag_1', + 'originalArray[0].tags[1]': 'tag_2', + 'originalArray[0].tags[2]': 'tag_3', + 'originalArray[1].nested_field': 'nested value', + 'originalArray[1].tags[0]': 'tag_1', + 'originalArray[2].nested_field': 'nested value', + }, + update_last_request_at: true, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'intercom', + description: 'Test 1', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + channel: 'mobile', + context: { + app: { + build: '1.0', + name: 'Test_Example', + namespace: 'com.example.testapp', + version: '1.0', + }, + device: { + id: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + manufacturer: 'Apple', + model: 'iPhone', + name: 'iPod touch (7th generation)', + type: 'iOS', + }, + library: { + name: 'test-ios-library', + version: '1.0.7', + }, + locale: 'en-US', + network: { + bluetooth: false, + carrier: 'unavailable', + cellular: false, + wifi: true, + }, + os: { + name: 'iOS', + version: '14.0', + }, + screen: { + density: 2, + height: 320, + width: 568, + }, + timezone: 'Asia/Kolkata', + traits: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + firstName: 'Test', + lastName: 'Name', + createdAt: '2020-09-30T19:11:00.337Z', + email: 'test_1@test.com', + phone: '9876543210', + key1: 'value1', + }, + userAgent: 'unknown', + }, + event: 'Test Event 2', + integrations: { + All: true, + }, + messageId: '1601493060-39010c49-e6e4-4626-a75c-0dbf1925c9e8', + originalTimestamp: '2020-09-30T19:11:00.337Z', + receivedAt: '2020-10-01T00:41:11.369+05:30', + request_ip: '2405:201:8005:9856:7911:25e7:5603:5e18', + sentAt: '2020-09-30T19:11:10.382Z', + timestamp: '2020-10-01T00:41:01.324+05:30', + type: 'identify', + }, + destination: { + Config: { + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.intercom.io/users', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer intercomApiKey', + Accept: 'application/json', + 'Intercom-Version': '1.4', + }, + params: {}, + body: { + JSON: { + email: 'test_1@test.com', + phone: '9876543210', + signed_up_at: 1601493060, + last_seen_user_agent: 'unknown', + custom_attributes: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + key1: 'value1', + }, + update_last_request_at: true, + name: 'Test Name', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'intercom', + description: 'Test 2', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + channel: 'mobile', + context: { + app: { + build: '1.0', + name: 'Test_Example', + namespace: 'com.example.testapp', + version: '1.0', + }, + device: { + id: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + manufacturer: 'Apple', + model: 'iPhone', + name: 'iPod touch (7th generation)', + type: 'iOS', + }, + library: { + name: 'test-ios-library', + version: '1.0.7', + }, + locale: 'en-US', + network: { + bluetooth: false, + carrier: 'unavailable', + cellular: false, + wifi: true, + }, + os: { + name: 'iOS', + version: '14.0', + }, + screen: { + density: 2, + height: 320, + width: 568, + }, + timezone: 'Asia/Kolkata', + traits: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + lastName: 'Name', + createdAt: '2020-09-30T19:11:00.337Z', + email: 'test_1@test.com', + phone: '9876543210', + key1: 'value1', + }, + userAgent: 'unknown', + }, + event: 'Test Event 2', + integrations: { + All: true, + }, + messageId: '1601493060-39010c49-e6e4-4626-a75c-0dbf1925c9e8', + originalTimestamp: '2020-09-30T19:11:00.337Z', + receivedAt: '2020-10-01T00:41:11.369+05:30', + request_ip: '2405:201:8005:9856:7911:25e7:5603:5e18', + sentAt: '2020-09-30T19:11:10.382Z', + timestamp: '2020-10-01T00:41:01.324+05:30', + type: 'identify', + }, + destination: { + Config: { + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.intercom.io/users', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer intercomApiKey', + Accept: 'application/json', + 'Intercom-Version': '1.4', + }, + params: {}, + body: { + JSON: { + email: 'test_1@test.com', + phone: '9876543210', + signed_up_at: 1601493060, + last_seen_user_agent: 'unknown', + custom_attributes: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + key1: 'value1', + }, + update_last_request_at: true, + name: 'Name', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + }, + statusCode: 200, + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/intercom/router/data.ts b/test/integrations/destinations/intercom/router/data.ts index 7ce3c7351a..2ce8621ca1 100644 --- a/test/integrations/destinations/intercom/router/data.ts +++ b/test/integrations/destinations/intercom/router/data.ts @@ -794,4 +794,394 @@ export const data = [ }, }, }, + { + name: 'intercom', + description: 'Test 0', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + channel: 'mobile', + context: { + app: { + build: '1.0', + name: 'Test_Example', + namespace: 'com.example.testapp', + version: '1.0', + }, + device: { + id: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + manufacturer: 'Apple', + model: 'iPhone', + name: 'iPod touch (7th generation)', + type: 'iOS', + }, + library: { + name: 'test-ios-library', + version: '1.0.7', + }, + locale: 'en-US', + network: { + bluetooth: false, + carrier: 'unavailable', + cellular: false, + wifi: true, + }, + os: { + name: 'iOS', + version: '14.0', + }, + screen: { + density: 2, + height: 320, + width: 568, + }, + timezone: 'Asia/Kolkata', + traits: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + name: 'Test Name', + firstName: 'Test', + lastName: 'Name', + createdAt: '2020-09-30T19:11:00.337Z', + userId: 'test_user_id_1', + email: 'test_1@test.com', + phone: '9876543210', + key1: 'value1', + }, + userAgent: 'unknown', + }, + event: 'Test Event 2', + integrations: { + All: true, + }, + messageId: '1601493060-39010c49-e6e4-4626-a75c-0dbf1925c9e8', + originalTimestamp: '2020-09-30T19:11:00.337Z', + receivedAt: '2020-10-01T00:41:11.369+05:30', + request_ip: '2405:201:8005:9856:7911:25e7:5603:5e18', + sentAt: '2020-09-30T19:11:10.382Z', + timestamp: '2020-10-01T00:41:01.324+05:30', + type: 'identify', + }, + metadata: { + jobId: 1, + }, + destination: { + Config: { + apiKey: 'testApiKey', + apiVersion: 'v1', + sendAnonymousId: false, + updateLastRequestAt: false, + collectContext: false, + }, + }, + }, + { + message: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + channel: 'mobile', + context: { + app: { + build: '1.0', + name: 'Test_Example', + namespace: 'com.example.testapp', + version: '1.0', + }, + device: { + id: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + manufacturer: 'Apple', + model: 'iPhone', + name: 'iPod touch (7th generation)', + type: 'iOS', + }, + library: { + name: 'test-ios-library', + version: '1.0.7', + }, + locale: 'en-US', + network: { + bluetooth: false, + carrier: 'unavailable', + cellular: false, + wifi: true, + }, + os: { + name: 'iOS', + version: '14.0', + }, + screen: { + density: 2, + height: 320, + width: 568, + }, + timezone: 'Asia/Kolkata', + traits: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + firstName: 'Test', + lastName: 'Name', + createdAt: '2020-09-30T19:11:00.337Z', + email: 'test_1@test.com', + phone: '9876543210', + key1: 'value1', + }, + userAgent: 'unknown', + }, + event: 'Test Event 2', + integrations: { + All: true, + }, + messageId: '1601493060-39010c49-e6e4-4626-a75c-0dbf1925c9e8', + originalTimestamp: '2020-09-30T19:11:00.337Z', + receivedAt: '2020-10-01T00:41:11.369+05:30', + request_ip: '2405:201:8005:9856:7911:25e7:5603:5e18', + sentAt: '2020-09-30T19:11:10.382Z', + timestamp: '2020-10-01T00:41:01.324+05:30', + type: 'identify', + }, + metadata: { + jobId: 2, + }, + destination: { + Config: { + apiKey: 'testApiKey', + apiVersion: 'v1', + sendAnonymousId: false, + updateLastRequestAt: false, + collectContext: false, + }, + }, + }, + { + message: { + userId: 'user@5', + groupId: 'rudderlabs', + channel: 'web', + context: { + traits: { + email: 'test+5@rudderlabs.com', + phone: '+91 9599999999', + firstName: 'John', + lastName: 'Snow', + ownerId: '17', + }, + }, + traits: { + name: 'RudderStack', + size: 500, + website: 'www.rudderstack.com', + industry: 'CDP', + plan: 'enterprise', + }, + type: 'group', + originalTimestamp: '2023-11-10T14:42:44.724Z', + timestamp: '2023-11-22T10:12:44.757+05:30', + }, + destination: { + Config: { + apiKey: 'testApiKey', + apiVersion: 'v1', + sendAnonymousId: false, + collectContext: false, + }, + }, + metadata: { + jobId: 3, + }, + }, + ], + destType: 'intercom', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.intercom.io/users', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer testApiKey', + Accept: 'application/json', + 'Intercom-Version': '1.4', + }, + params: {}, + body: { + JSON: { + email: 'test_1@test.com', + phone: '9876543210', + name: 'Test Name', + signed_up_at: 1601493060, + last_seen_user_agent: 'unknown', + update_last_request_at: false, + user_id: 'test_user_id_1', + custom_attributes: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + key1: 'value1', + }, + }, + XML: {}, + JSON_ARRAY: {}, + FORM: {}, + }, + files: {}, + userId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + }, + metadata: [ + { + jobId: 1, + }, + ], + batched: false, + statusCode: 200, + destination: { + Config: { + apiKey: 'testApiKey', + apiVersion: 'v1', + collectContext: false, + sendAnonymousId: false, + updateLastRequestAt: false, + }, + }, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.intercom.io/users', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer testApiKey', + Accept: 'application/json', + 'Intercom-Version': '1.4', + }, + params: {}, + body: { + JSON: { + email: 'test_1@test.com', + phone: '9876543210', + signed_up_at: 1601493060, + name: 'Test Name', + last_seen_user_agent: 'unknown', + update_last_request_at: false, + custom_attributes: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + key1: 'value1', + }, + }, + XML: {}, + JSON_ARRAY: {}, + FORM: {}, + }, + files: {}, + userId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + }, + metadata: [ + { + jobId: 2, + }, + ], + batched: false, + statusCode: 200, + destination: { + Config: { + apiKey: 'testApiKey', + apiVersion: 'v1', + collectContext: false, + sendAnonymousId: false, + updateLastRequestAt: false, + }, + }, + }, + { + batched: false, + batchedRequest: [ + { + body: { + FORM: {}, + JSON: { + company_id: 'rudderlabs', + industry: 'CDP', + name: 'RudderStack', + plan: 'enterprise', + size: 500, + website: 'www.rudderstack.com', + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.intercom.io/companies', + files: {}, + headers: { + Accept: 'application/json', + Authorization: 'Bearer testApiKey', + 'Content-Type': 'application/json', + 'Intercom-Version': '1.4', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + { + body: { + FORM: {}, + JSON: { + companies: [ + { + company_id: 'rudderlabs', + name: 'RudderStack', + }, + ], + email: 'test+5@rudderlabs.com', + user_id: 'user@5', + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.intercom.io/users', + files: {}, + headers: { + Accept: 'application/json', + Authorization: 'Bearer testApiKey', + 'Content-Type': 'application/json', + 'Intercom-Version': '1.4', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + ], + destination: { + Config: { + apiKey: 'testApiKey', + apiVersion: 'v1', + collectContext: false, + sendAnonymousId: false, + }, + }, + metadata: [ + { + jobId: 3, + }, + ], + statusCode: 200, + }, + ], + }, + }, + }, + }, ]; From a60694cef1da31d27a5cf90264548cad793f556f Mon Sep 17 00:00:00 2001 From: Mihir Bhalala <77438541+mihir-4116@users.noreply.github.com> Date: Wed, 7 Feb 2024 10:43:05 +0530 Subject: [PATCH 06/13] feat(hs): chunking data based on batch limit (#2907) * feat(hs): chunking data based on batch limit * fix: code review changes * chore: code review changes * chore: code review changes * chore: code review changes * chore: code review changes * chore: code review changes * fix(hs): test case response * chore: code refactor * refactor getExistingContactsData * chore: code refactor * refactor getExistingContactsData --------- Co-authored-by: Dilip Kola --- src/v0/destinations/hs/config.js | 3 + src/v0/destinations/hs/util.js | 225 ++++++++++++------ src/v0/destinations/hs/util.test.js | 191 ++++++++++++++- .../destinations/hs/processor/data.ts | 2 +- 4 files changed, 342 insertions(+), 79 deletions(-) diff --git a/src/v0/destinations/hs/config.js b/src/v0/destinations/hs/config.js index b602a7542f..fb9790f0e5 100644 --- a/src/v0/destinations/hs/config.js +++ b/src/v0/destinations/hs/config.js @@ -64,6 +64,8 @@ const API_VERSION = { v3: 'newApi', }; +const MAX_CONTACTS_PER_REQUEST = 100; + const ConfigCategory = { COMMON: { name: 'HSCommonConfig', @@ -109,5 +111,6 @@ module.exports = { SEARCH_LIMIT_VALUE, RETL_SOURCE, RETL_CREATE_ASSOCIATION_OPERATION, + MAX_CONTACTS_PER_REQUEST, DESTINATION: 'HS', }; diff --git a/src/v0/destinations/hs/util.js b/src/v0/destinations/hs/util.js index 5c8f4a908a..e905ee63c4 100644 --- a/src/v0/destinations/hs/util.js +++ b/src/v0/destinations/hs/util.js @@ -1,3 +1,5 @@ +/* eslint-disable no-await-in-loop */ +const lodash = require('lodash'); const get = require('get-value'); const { NetworkInstrumentationError, @@ -25,6 +27,7 @@ const { SEARCH_LIMIT_VALUE, hsCommonConfigJson, DESTINATION, + MAX_CONTACTS_PER_REQUEST, } = require('./config'); const tags = require('../../util/tags'); @@ -464,42 +467,127 @@ const getEventAndPropertiesFromConfig = (message, destination, payload) => { }; /** - * DOC: https://developers.hubspot.com/docs/api/crm/search + * Validates object and identifier type is present in message + * @param {*} firstMessage + * @returns + */ +const getObjectAndIdentifierType = (firstMessage) => { + const { objectType, identifierType } = getDestinationExternalIDInfoForRetl( + firstMessage, + DESTINATION, + ); + if (!objectType || !identifierType) { + throw new InstrumentationError('rETL - external Id not found.'); + } + return { objectType, identifierType }; +}; + +/** + * Returns values for search api call * @param {*} inputs + * @returns + */ +const extractIDsForSearchAPI = (inputs) => { + const values = inputs.map((input) => { + const { message } = input; + const { destinationExternalId } = getDestinationExternalIDInfoForRetl(message, DESTINATION); + return destinationExternalId.toString().toLowerCase(); + }); + + return Array.from(new Set(values)); +}; + +/** + * Returns hubspot records + * Ref : https://developers.hubspot.com/docs/api/crm/search + * @param {*} data + * @param {*} requestOptions + * @param {*} objectType + * @param {*} identifierType * @param {*} destination + * @returns */ -const getExistingData = async (inputs, destination) => { +const performHubSpotSearch = async ( + reqdata, + reqOptions, + objectType, + identifierType, + destination, +) => { + let checkAfter = 1; + const searchResults = []; + const requestData = reqdata; const { Config } = destination; - let values = []; - let searchResponse; - let updateHubspotIds = []; - const firstMessage = inputs[0].message; - let objectType = null; - let identifierType = null; - - if (firstMessage) { - objectType = getDestinationExternalIDInfoForRetl(firstMessage, DESTINATION).objectType; - identifierType = getDestinationExternalIDInfoForRetl(firstMessage, DESTINATION).identifierType; - if (!objectType || !identifierType) { - throw new InstrumentationError('rETL - external Id not found.'); + + const endpoint = IDENTIFY_CRM_SEARCH_ALL_OBJECTS.replace(':objectType', objectType); + const endpointPath = `objects/:objectType/search`; + + const url = + Config.authorizationType === 'newPrivateAppApi' + ? endpoint + : `${endpoint}?hapikey=${Config.apiKey}`; + + const requestOptions = Config.authorizationType === 'newPrivateAppApi' ? reqOptions : {}; + + /* * + * This is needed for processing paginated response when searching hubspot. + * we can't avoid await in loop as response to the request contains the pagination details + * */ + + while (checkAfter) { + const searchResponse = await httpPOST(url, requestData, requestOptions, { + destType: 'hs', + feature: 'transformation', + endpointPath, + }); + + const processedResponse = processAxiosResponse(searchResponse); + + if (processedResponse.status !== 200) { + throw new NetworkError( + `rETL - Error during searching object record. ${JSON.stringify( + processedResponse.response?.message, + )}`, + processedResponse.status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(processedResponse.status), + }, + processedResponse, + ); + } + + const after = processedResponse.response?.paging?.next?.after || 0; + requestData.after = after; // assigning to the new value of after + checkAfter = after; // assigning to the new value if no after we assign it to 0 and no more calls will take place + + const results = processedResponse.response?.results; + if (results) { + searchResults.push( + ...results.map((result) => ({ + id: result.id, + property: result.properties[identifierType], + })), + ); } - } else { - throw new InstrumentationError('rETL - objectType or identifier type not found. '); } - inputs.map(async (input) => { - const { message } = input; - const { destinationExternalId } = getDestinationExternalIDInfoForRetl(message, DESTINATION); - values.push(destinationExternalId.toString().toLowerCase()); - }); - values = Array.from(new Set(values)); + return searchResults; +}; + +/** + * Returns requestData + * @param {*} identifierType + * @param {*} chunk + * @returns + */ +const getRequestData = (identifierType, chunk) => { const requestData = { filterGroups: [ { filters: [ { propertyName: identifierType, - values, + values: chunk, operator: 'IN', }, ], @@ -510,65 +598,45 @@ const getExistingData = async (inputs, destination) => { after: 0, }; + return requestData; +}; + +/** + * DOC: https://developers.hubspot.com/docs/api/crm/search + * @param {*} inputs + * @param {*} destination + */ +const getExistingContactsData = async (inputs, destination) => { + const { Config } = destination; + const updateHubspotIds = []; + const firstMessage = inputs[0].message; + + if (!firstMessage) { + throw new InstrumentationError('rETL - objectType or identifier type not found.'); + } + + const { objectType, identifierType } = getObjectAndIdentifierType(firstMessage); + + const values = extractIDsForSearchAPI(inputs); + const valuesChunk = lodash.chunk(values, MAX_CONTACTS_PER_REQUEST); const requestOptions = { headers: { 'Content-Type': JSON_MIME_TYPE, Authorization: `Bearer ${Config.accessToken}`, }, }; - let checkAfter = 1; // variable to keep checking if we have more results - - /* eslint-disable no-await-in-loop */ - - /* * - * This is needed for processing paginated response when searching hubspot. - * we can't avoid await in loop as response to the request contains the pagination details - * */ - - while (checkAfter) { - const endpoint = IDENTIFY_CRM_SEARCH_ALL_OBJECTS.replace(':objectType', objectType); - const endpointPath = `objects/:objectType/search`; - - const url = - Config.authorizationType === 'newPrivateAppApi' - ? endpoint - : `${endpoint}?hapikey=${Config.apiKey}`; - searchResponse = - Config.authorizationType === 'newPrivateAppApi' - ? await httpPOST(url, requestData, requestOptions, { - destType: 'hs', - feature: 'transformation', - endpointPath, - }) - : await httpPOST(url, requestData, { - destType: 'hs', - feature: 'transformation', - endpointPath, - }); - searchResponse = processAxiosResponse(searchResponse); - - if (searchResponse.status !== 200) { - throw new NetworkError( - `rETL - Error during searching object record. ${searchResponse.response?.message}`, - searchResponse.status, - { - [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(searchResponse.status), - }, - searchResponse, - ); - } - - const after = searchResponse.response?.paging?.next?.after || 0; - - requestData.after = after; // assigning to the new value of after - checkAfter = after; // assigning to the new value if no after we assign it to 0 and no more calls will take place - - const results = searchResponse.response?.results; - if (results) { - updateHubspotIds = results.map((result) => { - const propertyValue = result.properties[identifierType]; - return { id: result.id, property: propertyValue }; - }); + // eslint-disable-next-line no-restricted-syntax + for (const chunk of valuesChunk) { + const requestData = getRequestData(identifierType, chunk); + const searchResults = await performHubSpotSearch( + requestData, + requestOptions, + objectType, + identifierType, + destination, + ); + if (searchResults.length > 0) { + updateHubspotIds.push(...searchResults); } } return updateHubspotIds; @@ -601,7 +669,7 @@ const setHsSearchId = (input, id) => { const splitEventsForCreateUpdate = async (inputs, destination) => { // get all the id and properties of already existing objects needed for update. - const updateHubspotIds = await getExistingData(inputs, destination); + const updateHubspotIds = await getExistingContactsData(inputs, destination); const resultInput = inputs.map((input) => { const { message } = input; @@ -680,4 +748,7 @@ module.exports = { validatePayloadDataTypes, getUTCMidnightTimeStampValue, populateTraits, + getObjectAndIdentifierType, + extractIDsForSearchAPI, + getRequestData, }; diff --git a/src/v0/destinations/hs/util.test.js b/src/v0/destinations/hs/util.test.js index 737b206401..30e89d3aee 100644 --- a/src/v0/destinations/hs/util.test.js +++ b/src/v0/destinations/hs/util.test.js @@ -1,4 +1,9 @@ -const { validatePayloadDataTypes } = require('../../../../src/v0/destinations/hs/util'); +const { + getRequestData, + extractIDsForSearchAPI, + validatePayloadDataTypes, + getObjectAndIdentifierType, +} = require('./util'); const propertyMap = { firstName: 'string', @@ -40,3 +45,187 @@ describe('Validate payload data types utility function test cases', () => { } }); }); + +describe('getObjectAndIdentifierType utility test cases', () => { + it('should return an object with objectType and identifierType properties when given a valid input', () => { + const firstMessage = { + type: 'identify', + traits: { + to: { + id: 1, + }, + from: { + id: 940, + }, + }, + userId: '1', + context: { + externalId: [ + { + id: 1, + type: 'HS-association', + toObjectType: 'contacts', + fromObjectType: 'companies', + identifierType: 'id', + associationTypeId: 'engineer', + }, + ], + mappedToDestination: 'true', + }, + }; + const result = getObjectAndIdentifierType(firstMessage); + expect(result).toEqual({ objectType: 'association', identifierType: 'id' }); + }); + + it('should throw an error when objectType or identifierType is not present in input', () => { + const firstMessage = { + type: 'identify', + traits: { + to: { + id: 1, + }, + from: { + id: 940, + }, + }, + userId: '1', + context: { + externalId: [ + { + id: 1, + type: 'HS-', + toObjectType: 'contacts', + fromObjectType: 'companies', + associationTypeId: 'engineer', + }, + ], + mappedToDestination: 'true', + }, + }; + try { + getObjectAndIdentifierType(firstMessage); + } catch (err) { + expect(err.message).toBe('rETL - external Id not found.'); + } + }); +}); + +describe('extractUniqueValues utility test cases', () => { + it('Should return an array of unique values', () => { + const inputs = [ + { + message: { + context: { + externalId: [ + { + identifierType: 'email', + id: 'testhubspot2@email.com', + type: 'HS-lead', + }, + ], + mappedToDestination: true, + }, + }, + }, + { + message: { + context: { + externalId: [ + { + identifierType: 'email', + id: 'Testhubspot3@email.com', + type: 'HS-lead', + }, + ], + mappedToDestination: true, + }, + }, + }, + { + message: { + context: { + externalId: [ + { + identifierType: 'email', + id: 'testhubspot4@email.com', + type: 'HS-lead', + }, + ], + mappedToDestination: true, + }, + }, + }, + { + message: { + context: { + externalId: [ + { + identifierType: 'email', + id: 'testHUBSPOT5@email.com', + type: 'HS-lead', + }, + ], + mappedToDestination: true, + }, + }, + }, + { + message: { + context: { + externalId: [ + { + identifierType: 'email', + id: 'testhubspot2@email.com', + type: 'HS-lead', + }, + ], + mappedToDestination: true, + }, + }, + }, + ]; + + const result = extractIDsForSearchAPI(inputs); + + expect(result).toEqual([ + 'testhubspot2@email.com', + 'testhubspot3@email.com', + 'testhubspot4@email.com', + 'testhubspot5@email.com', + ]); + }); + + it('Should return an empty array when the input is empty', () => { + const inputs = []; + const result = extractIDsForSearchAPI(inputs); + expect(result).toEqual([]); + }); +}); + +describe('getRequestDataAndRequestOptions utility test cases', () => { + it('Should return an object with requestData and requestOptions', () => { + const identifierType = 'email'; + const chunk = 'test1@gmail.com'; + const accessToken = 'dummyAccessToken'; + + const expectedRequestData = { + filterGroups: [ + { + filters: [ + { + propertyName: identifierType, + values: chunk, + operator: 'IN', + }, + ], + }, + ], + properties: [identifierType], + limit: 100, + after: 0, + }; + + const requestData = getRequestData(identifierType, chunk, accessToken); + expect(requestData).toEqual(expectedRequestData); + }); +}); diff --git a/test/integrations/destinations/hs/processor/data.ts b/test/integrations/destinations/hs/processor/data.ts index 5eaa109dc4..03ad9d0a3b 100644 --- a/test/integrations/destinations/hs/processor/data.ts +++ b/test/integrations/destinations/hs/processor/data.ts @@ -4769,7 +4769,7 @@ export const data = [ body: [ { error: - '{"message":"rETL - Error during searching object record. Request Rate Limit reached","destinationResponse":{"response":{"status":"error","message":"Request Rate Limit reached","correlationId":"4d39ff11-e121-4514-bcd8-132a9dd1ff50","category":"RATE-LIMIT_REACHED","links":{"api key":"https://app.hubspot.com/l/api-key/"}},"status":429}}', + '{"message":"rETL - Error during searching object record. \\"Request Rate Limit reached\\"","destinationResponse":{"response":{"status":"error","message":"Request Rate Limit reached","correlationId":"4d39ff11-e121-4514-bcd8-132a9dd1ff50","category":"RATE-LIMIT_REACHED","links":{"api key":"https://app.hubspot.com/l/api-key/"}},"status":429}}', metadata: { jobId: 2, }, From 8a8db1e8bf230731c675fe16798280e0053ded9e Mon Sep 17 00:00:00 2001 From: Dilip Kola Date: Mon, 12 Feb 2024 16:27:22 +0530 Subject: [PATCH 07/13] chore: upgrade workflow engine to 0.7.2 --- .gitignore | 2 +- package-lock.json | 16 ++++++++-------- package.json | 2 +- src/cdk/v2/handler.ts | 15 +++++++++++---- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index f96c3ac807..956605f139 100644 --- a/.gitignore +++ b/.gitignore @@ -132,7 +132,7 @@ dist # Others **/.DS_Store - +.dccache .idea diff --git a/package-lock.json b/package-lock.json index dd42fd3921..d32e378d2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.6", "@rudderstack/integrations-lib": "^0.2.2", - "@rudderstack/workflow-engine": "^0.6.9", + "@rudderstack/workflow-engine": "^0.7.2", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", "ajv-formats": "^2.1.1", @@ -4535,17 +4535,17 @@ } }, "node_modules/@rudderstack/json-template-engine": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.8.2.tgz", - "integrity": "sha512-9oMBnqgNuwiXd7MUlNOAchCnJXQAy6w6XGmDqDM6iXdYDkvqYFiq7sbg5j4SdtpTTST293hahREr5PXfFVzVKg==" + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.8.5.tgz", + "integrity": "sha512-+iH40g+ZA2ANgwjOITdEdZJLZV+ljR28Akn/dRoDia591tMu7PptyvDaAvl+m1DijWXddpLQ8SX9xaEcIdmqlw==" }, "node_modules/@rudderstack/workflow-engine": { - "version": "0.6.10", - "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.6.10.tgz", - "integrity": "sha512-3GRdnbB0BuSPWiKf4JsSpG7QuGffAFWkT5T0JLR7Jxps25gt+PgtjQiAlwrRhO5A0WeTJMIKTI7ctz6dGmJosg==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.7.2.tgz", + "integrity": "sha512-aXQvoXMekvXxxDG6Yc5P5l3PJIwqVA+EmJ2w4SnQ94BUHhbsybPjgGvyzD17MUTAdWEOtqS38SuzLflBs/5T4g==", "dependencies": { "@aws-crypto/sha256-js": "^5.0.0", - "@rudderstack/json-template-engine": "^0.8.1", + "@rudderstack/json-template-engine": "^0.8.4", "jsonata": "^2.0.3", "lodash": "^4.17.21", "object-sizeof": "^2.6.3", diff --git a/package.json b/package.json index ac6746ed20..0d8e528342 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.6", "@rudderstack/integrations-lib": "^0.2.2", - "@rudderstack/workflow-engine": "^0.6.9", + "@rudderstack/workflow-engine": "^0.7.2", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", "ajv-formats": "^2.1.1", diff --git a/src/cdk/v2/handler.ts b/src/cdk/v2/handler.ts index 47d6d10179..edd14e7298 100644 --- a/src/cdk/v2/handler.ts +++ b/src/cdk/v2/handler.ts @@ -50,16 +50,20 @@ export async function getWorkflowEngine( const workflowEnginePromiseMap = new Map(); -export function getCachedWorkflowEngine( +export async function getCachedWorkflowEngine( destName: string, feature: string, bindings: Record = {}, -): WorkflowEngine { +): Promise { // Create a new instance of the engine for the destination if needed // TODO: Use cache to avoid long living engine objects workflowEnginePromiseMap[destName] = workflowEnginePromiseMap[destName] || new Map(); if (!workflowEnginePromiseMap[destName][feature]) { - workflowEnginePromiseMap[destName][feature] = getWorkflowEngine(destName, feature, bindings); + workflowEnginePromiseMap[destName][feature] = await getWorkflowEngine( + destName, + feature, + bindings, + ); } return workflowEnginePromiseMap[destName][feature]; } @@ -97,5 +101,8 @@ export function executeStep( ): Promise { return workflowEngine .getStepExecutor(stepName) - .execute(input, Object.assign(workflowEngine.bindings, getEmptyExecutionBindings(), bindings)); + .execute( + input, + Object.assign(workflowEngine.getBindings(), getEmptyExecutionBindings(), bindings), + ); } From 69c83489c85486c9b2aed4a1292bd9f0aae9ca44 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Mon, 12 Feb 2024 19:07:26 +0530 Subject: [PATCH 08/13] fix: amplitude batch output metadata (#3077) * fix: amplitude batching * refactor: remove commented code * refactor: remove redundant import --- src/v0/destinations/am/transform.js | 13 +- .../destination/batch/failure_batch.json | 403 +++++++++++++----- .../destinations/am/batch/data.ts | 132 +++--- 3 files changed, 369 insertions(+), 179 deletions(-) diff --git a/src/v0/destinations/am/transform.js b/src/v0/destinations/am/transform.js index 911ec51be0..d78a5f727f 100644 --- a/src/v0/destinations/am/transform.js +++ b/src/v0/destinations/am/transform.js @@ -22,13 +22,13 @@ const { getFieldValueFromMessage, getValueFromMessage, deleteObjectProperty, - getErrorRespEvents, removeUndefinedAndNullValues, isDefinedAndNotNull, isAppleFamily, isDefinedAndNotNullAndNotEmpty, simpleProcessRouterDest, isValidInteger, + handleRtTfSingleEventError, } = require('../../util'); const { BASE_URL, @@ -40,7 +40,6 @@ const { AMBatchSizeLimit, AMBatchEventLimit, } = require('./config'); -const tags = require('../../util/tags'); const AMUtils = require('./utils'); @@ -904,16 +903,10 @@ const batch = (destEvents) => { // this case shold not happen and should be filtered already // by the first pass of single event transformation if (messageEvent && !userId && !deviceId) { - const errorResponse = getErrorRespEvents( - metadata, - 400, + const MissingUserIdDeviceIdError = new InstrumentationError( 'Both userId and deviceId cannot be undefined', - { - [tags.TAG_NAMES.ERROR_CATEGORY]: tags.ERROR_CATEGORIES.DATA_VALIDATION, - [tags.TAG_NAMES.ERROR_TYPE]: tags.ERROR_TYPES.INSTRUMENTATION, - }, ); - respList.push(errorResponse); + respList.push(handleRtTfSingleEventError(ev, MissingUserIdDeviceIdError, {})); return; } /* check if not a JSON body or (userId length < 5 && batchEventsWithUserIdLengthLowerThanFive is false) or diff --git a/test/apitests/data_scenarios/destination/batch/failure_batch.json b/test/apitests/data_scenarios/destination/batch/failure_batch.json index 8063bc74a1..6352ca1a11 100644 --- a/test/apitests/data_scenarios/destination/batch/failure_batch.json +++ b/test/apitests/data_scenarios/destination/batch/failure_batch.json @@ -1051,125 +1051,314 @@ }, "output": [ { - "metadata": { - "userId": "<<>>testUser<<>>testUser", - "jobId": 2, - "sourceId": "27O0bmEEx3GgfmEhZHUcPwJQVWC", - "destinationId": "2JK3ACpBjq9AmvUbxR1u2pDPSYR", - "attemptNum": 0, - "receivedAt": "2022-12-24T17:29:00.699+05:30", - "createdAt": "2022-12-24T11:59:03.125Z", - "firstAttemptedAt": "", - "transformAt": "processor", - "workspaceId": "27O0bhB6p5ehfOWeeZlOSsSDTLg", - "secret": null, - "jobsT": { - "UUID": "aaa8b7c4-2600-478b-b275-01740e1ef50c", - "JobID": 2, - "UserID": "<<>>testUser<<>>testUser", - "CreatedAt": "2022-12-24T11:59:03.125515Z", - "ExpireAt": "2022-12-24T11:59:03.125515Z", - "CustomVal": "AM", - "EventCount": 1, - "EventPayload": { - "body": { - "XML": {}, - "FORM": {}, - "JSON": { - "events": [ - { - "ip": "[::1]", - "time": 1671883143047, - "library": "rudderstack", - "user_id": "testUser", - "device_id": "anon-id", - "insert_id": "14642496-9a12-4db7-b0f2-9a336cf6cea9", - "event_type": "Product Added", - "session_id": -1, - "user_properties": { - "email": "test.c97@gmail.com", - "phone": "+919876543210", - "gender": "Male", - "lastName": "Rudderlabs", - "firstName": "test" - }, - "event_properties": { - "sku": "F15", - "url": "https://www.website.com/product/path", - "name": "Game", - "brand": "Gamepro", - "price": 13.49, - "coupon": "DISC21", - "variant": "111", - "category": "Games", - "position": 1, - "quantity": 11, - "image_url": "https://www.website.com/product/path.png", - "product_id": "123" + "metadata": [ + { + "userId": "<<>>testUser<<>>testUser", + "jobId": 2, + "sourceId": "27O0bmEEx3GgfmEhZHUcPwJQVWC", + "destinationId": "2JK3ACpBjq9AmvUbxR1u2pDPSYR", + "attemptNum": 0, + "receivedAt": "2022-12-24T17:29:00.699+05:30", + "createdAt": "2022-12-24T11:59:03.125Z", + "firstAttemptedAt": "", + "transformAt": "processor", + "workspaceId": "27O0bhB6p5ehfOWeeZlOSsSDTLg", + "secret": null, + "jobsT": { + "UUID": "aaa8b7c4-2600-478b-b275-01740e1ef50c", + "JobID": 2, + "UserID": "<<>>testUser<<>>testUser", + "CreatedAt": "2022-12-24T11:59:03.125515Z", + "ExpireAt": "2022-12-24T11:59:03.125515Z", + "CustomVal": "AM", + "EventCount": 1, + "EventPayload": { + "body": { + "XML": {}, + "FORM": {}, + "JSON": { + "events": [ + { + "ip": "[::1]", + "time": 1671883143047, + "library": "rudderstack", + "user_id": "testUser", + "device_id": "anon-id", + "insert_id": "14642496-9a12-4db7-b0f2-9a336cf6cea9", + "event_type": "Product Added", + "session_id": -1, + "user_properties": { + "email": "test.c97@gmail.com", + "phone": "+919876543210", + "gender": "Male", + "lastName": "Rudderlabs", + "firstName": "test" + }, + "event_properties": { + "sku": "F15", + "url": "https://www.website.com/product/path", + "name": "Game", + "brand": "Gamepro", + "price": 13.49, + "coupon": "DISC21", + "variant": "111", + "category": "Games", + "position": 1, + "quantity": 11, + "image_url": "https://www.website.com/product/path.png", + "product_id": "123" + } } + ], + "api_key": "dummyApiKey", + "options": { + "min_id_length": 1 } - ], - "api_key": "dummyApiKey", - "options": { - "min_id_length": 1 - } + }, + "JSON_ARRAY": {} }, - "JSON_ARRAY": {} + "type": "REST", + "files": {}, + "method": "POST", + "params": {}, + "userId": "anon-id", + "headers": { + "Content-Type": "application/json" + }, + "version": "1", + "endpoint": "https://api2.amplitude.com/2/httpapi" }, - "type": "REST", - "files": {}, - "method": "POST", - "params": {}, - "userId": "anon-id", - "headers": { - "Content-Type": "application/json" + "PayloadSize": 1133, + "LastJobStatus": { + "JobID": 0, + "JobState": "", + "AttemptNum": 0, + "ExecTime": "0001-01-01T00:00:00Z", + "RetryTime": "0001-01-01T00:00:00Z", + "ErrorCode": "", + "ErrorResponse": null, + "Parameters": null, + "WorkspaceId": "" }, - "version": "1", - "endpoint": "https://api2.amplitude.com/2/httpapi" - }, - "PayloadSize": 1133, - "LastJobStatus": { - "JobID": 0, - "JobState": "", - "AttemptNum": 0, - "ExecTime": "0001-01-01T00:00:00Z", - "RetryTime": "0001-01-01T00:00:00Z", - "ErrorCode": "", - "ErrorResponse": null, - "Parameters": null, - "WorkspaceId": "" - }, - "Parameters": { - "record_id": null, - "source_id": "27O0bmEEx3GgfmEhZHUcPwJQVWC", - "event_name": "Product Added", - "event_type": "track", - "message_id": "14642496-9a12-4db7-b0f2-9a336cf6cea9", - "received_at": "2022-12-24T17:29:00.699+05:30", - "workspaceId": "27O0bhB6p5ehfOWeeZlOSsSDTLg", - "transform_at": "processor", - "source_job_id": "", - "destination_id": "2JK3ACpBjq9AmvUbxR1u2pDPSYR", - "gateway_job_id": 1, - "source_task_id": "", - "source_batch_id": "", - "source_category": "", - "source_job_run_id": "", - "source_task_run_id": "", - "source_definition_id": "1b6gJdqOPOCadT3cddw8eidV591", - "destination_definition_id": "" + "Parameters": { + "record_id": null, + "source_id": "27O0bmEEx3GgfmEhZHUcPwJQVWC", + "event_name": "Product Added", + "event_type": "track", + "message_id": "14642496-9a12-4db7-b0f2-9a336cf6cea9", + "received_at": "2022-12-24T17:29:00.699+05:30", + "workspaceId": "27O0bhB6p5ehfOWeeZlOSsSDTLg", + "transform_at": "processor", + "source_job_id": "", + "destination_id": "2JK3ACpBjq9AmvUbxR1u2pDPSYR", + "gateway_job_id": 1, + "source_task_id": "", + "source_batch_id": "", + "source_category": "", + "source_job_run_id": "", + "source_task_run_id": "", + "source_definition_id": "1b6gJdqOPOCadT3cddw8eidV591", + "destination_definition_id": "" + }, + "WorkspaceId": "27O0bhB6p5ehfOWeeZlOSsSDTLg" }, - "WorkspaceId": "27O0bhB6p5ehfOWeeZlOSsSDTLg" - }, - "workerAssignedTime": "2022-12-24T17:29:04.051596+05:30" - }, + "workerAssignedTime": "2022-12-24T17:29:04.051596+05:30" + } + ], "batched": false, "statusCode": 400, "statTags": { "errorCategory": "dataValidation", "errorType": "instrumentation" }, - "error": "Both userId and deviceId cannot be undefined" + "error": "Both userId and deviceId cannot be undefined", + "destination": { + "ID": "2JK3ACpBjq9AmvUbxR1u2pDPSYR", + "Name": "Amplitude-2", + "DestinationDefinition": { + "ID": "1QGzO4fWSyq3lsyFHf4eQAMDSr9", + "Name": "AM", + "DisplayName": "Amplitude", + "Config": { + "destConfig": { + "android": [ + "eventUploadPeriodMillis", + "eventUploadThreshold", + "useNativeSDK", + "enableLocationListening", + "trackSessionEvents", + "useAdvertisingIdForDeviceId" + ], + "defaultConfig": [ + "apiKey", + "groupTypeTrait", + "groupValueTrait", + "trackAllPages", + "trackCategorizedPages", + "trackNamedPages", + "traitsToIncrement", + "traitsToSetOnce", + "traitsToAppend", + "traitsToPrepend", + "trackProductsOnce", + "trackRevenuePerProduct", + "versionName", + "apiSecret", + "residencyServer", + "blacklistedEvents", + "whitelistedEvents", + "eventFilteringOption", + "mapDeviceBrand" + ], + "flutter": [ + "eventUploadPeriodMillis", + "eventUploadThreshold", + "useNativeSDK", + "enableLocationListening", + "trackSessionEvents", + "useAdvertisingIdForDeviceId", + "useIdfaAsDeviceId" + ], + "ios": [ + "eventUploadPeriodMillis", + "eventUploadThreshold", + "useNativeSDK", + "trackSessionEvents", + "useIdfaAsDeviceId" + ], + "reactnative": [ + "eventUploadPeriodMillis", + "eventUploadThreshold", + "useNativeSDK", + "enableLocationListening", + "trackSessionEvents", + "useAdvertisingIdForDeviceId", + "useIdfaAsDeviceId" + ], + "web": [ + "useNativeSDK", + "preferAnonymousIdForDeviceId", + "deviceIdFromUrlParam", + "forceHttps", + "trackGclid", + "trackReferrer", + "saveParamsReferrerOncePerSession", + "trackUtmProperties", + "unsetParamsReferrerOnNewSession", + "batchEvents", + "eventUploadPeriodMillis", + "eventUploadThreshold", + "oneTrustCookieCategories" + ] + }, + "excludeKeys": [], + "includeKeys": [ + "apiKey", + "groupTypeTrait", + "groupValueTrait", + "trackAllPages", + "trackCategorizedPages", + "trackNamedPages", + "traitsToIncrement", + "traitsToSetOnce", + "traitsToAppend", + "traitsToPrepend", + "trackProductsOnce", + "trackRevenuePerProduct", + "preferAnonymousIdForDeviceId", + "deviceIdFromUrlParam", + "forceHttps", + "trackGclid", + "trackReferrer", + "saveParamsReferrerOncePerSession", + "trackUtmProperties", + "unsetParamsReferrerOnNewSession", + "batchEvents", + "eventUploadPeriodMillis", + "eventUploadThreshold", + "versionName", + "enableLocationListening", + "useAdvertisingIdForDeviceId", + "trackSessionEvents", + "useIdfaAsDeviceId", + "blacklistedEvents", + "whitelistedEvents", + "oneTrustCookieCategories", + "eventFilteringOption", + "mapDeviceBrand" + ], + "saveDestinationResponse": true, + "secretKeys": ["apiKey", "apiSecret"], + "supportedMessageTypes": ["alias", "group", "identify", "page", "screen", "track"], + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova" + ], + "supportsVisualMapper": true, + "transformAt": "processor", + "transformAtV1": "processor" + }, + "ResponseRules": null + }, + "Config": { + "apiKey": "dummyApiKey", + "apiSecret": "", + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "eventFilteringOption": "disable", + "groupTypeTrait": "", + "groupValueTrait": "", + "mapDeviceBrand": false, + "residencyServer": "standard", + "trackAllPages": false, + "trackCategorizedPages": true, + "trackNamedPages": true, + "trackProductsOnce": false, + "trackRevenuePerProduct": false, + "traitsToAppend": [ + { + "traits": "" + } + ], + "traitsToIncrement": [ + { + "traits": "" + } + ], + "traitsToPrepend": [ + { + "traits": "" + } + ], + "traitsToSetOnce": [ + { + "traits": "" + } + ], + "versionName": "", + "whitelistedEvents": [ + { + "eventName": "" + } + ] + }, + "Enabled": true, + "WorkspaceID": "27O0bhB6p5ehfOWeeZlOSsSDTLg", + "Transformations": [], + "IsProcessorEnabled": true, + "RevisionID": "2JMKUgZX3b8sbtDSZrkUB7okeOY" + } }, { "batchedRequest": { diff --git a/test/integrations/destinations/am/batch/data.ts b/test/integrations/destinations/am/batch/data.ts index aa67df06c7..91a17606a9 100644 --- a/test/integrations/destinations/am/batch/data.ts +++ b/test/integrations/destinations/am/batch/data.ts @@ -61,7 +61,7 @@ export const data = [ endpoint: 'https://api.eu.amplitude.com/2/httpapi', }, metadata: { - job_id: 1, + jobId: 1, userId: 'u1', }, destination: { @@ -83,16 +83,24 @@ export const data = [ { batched: false, error: 'Both userId and deviceId cannot be undefined', - //TODO fix this - metadata: { - job_id: 1, - userId: 'u1', - }, + metadata: [ + { + jobId: 1, + userId: 'u1', + }, + ], statTags: { errorCategory: 'dataValidation', errorType: 'instrumentation', }, statusCode: 400, + destination: { + ID: 'a', + url: 'a', + Config: { + residencyServer: 'EU', + }, + }, }, ], }, @@ -163,7 +171,7 @@ export const data = [ endpoint: 'https://api.eu.amplitude.com/2/httpapi', }, metadata: { - job_id: 1, + jobId: 1, userId: 'u1', }, destination: { @@ -218,7 +226,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 2, + jobId: 2, userId: 'u1', }, destination: { @@ -273,7 +281,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 3, + jobId: 3, userId: 'u1', }, destination: { @@ -331,7 +339,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 4, + jobId: 4, userId: 'u1', }, destination: { @@ -389,7 +397,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 5, + jobId: 5, userId: 'u1', }, destination: { @@ -423,7 +431,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/groupidentify', }, metadata: { - job_id: 6, + jobId: 6, userId: 'u1', }, destination: { @@ -455,7 +463,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/usermap', }, metadata: { - job_id: 7, + jobId: 7, userId: 'u1', }, destination: { @@ -529,7 +537,7 @@ export const data = [ }, metadata: [ { - job_id: 1, + jobId: 1, userId: 'u1', }, ], @@ -565,7 +573,7 @@ export const data = [ }, metadata: [ { - job_id: 6, + jobId: 6, userId: 'u1', }, ], @@ -599,7 +607,7 @@ export const data = [ }, metadata: [ { - job_id: 7, + jobId: 7, userId: 'u1', }, ], @@ -710,19 +718,19 @@ export const data = [ }, metadata: [ { - job_id: 2, + jobId: 2, userId: 'u1', }, { - job_id: 3, + jobId: 3, userId: 'u1', }, { - job_id: 4, + jobId: 4, userId: 'u1', }, { - job_id: 5, + jobId: 5, userId: 'u1', }, ], @@ -801,7 +809,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 1, + jobId: 1, userId: 'u1', }, destination: { @@ -854,7 +862,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 2, + jobId: 2, userId: 'u1', }, destination: { @@ -907,7 +915,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 3, + jobId: 3, userId: 'u1', }, destination: { @@ -963,7 +971,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 4, + jobId: 4, userId: 'u1', }, destination: { @@ -1019,7 +1027,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 5, + jobId: 5, userId: 'u1', }, destination: { @@ -1053,7 +1061,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/groupidentify', }, metadata: { - job_id: 6, + jobId: 6, userId: 'u1', }, destination: { @@ -1085,7 +1093,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/usermap', }, metadata: { - job_id: 7, + jobId: 7, userId: 'u1', }, destination: { @@ -1157,7 +1165,7 @@ export const data = [ }, metadata: [ { - job_id: 1, + jobId: 1, userId: 'u1', }, ], @@ -1193,7 +1201,7 @@ export const data = [ }, metadata: [ { - job_id: 6, + jobId: 6, userId: 'u1', }, ], @@ -1227,7 +1235,7 @@ export const data = [ }, metadata: [ { - job_id: 7, + jobId: 7, userId: 'u1', }, ], @@ -1338,19 +1346,19 @@ export const data = [ }, metadata: [ { - job_id: 2, + jobId: 2, userId: 'u1', }, { - job_id: 3, + jobId: 3, userId: 'u1', }, { - job_id: 4, + jobId: 4, userId: 'u1', }, { - job_id: 5, + jobId: 5, userId: 'u1', }, ], @@ -2117,7 +2125,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 1, + jobId: 1, userId: 'u1', }, destination: { @@ -2172,7 +2180,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 2, + jobId: 2, userId: 'u1', }, destination: { @@ -2227,7 +2235,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 3, + jobId: 3, userId: 'u1', }, destination: { @@ -2285,7 +2293,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 4, + jobId: 4, userId: 'u1', }, destination: { @@ -2343,7 +2351,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 5, + jobId: 5, userId: 'u1', }, destination: { @@ -2377,7 +2385,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/groupidentify', }, metadata: { - job_id: 6, + jobId: 6, userId: 'u1', }, destination: { @@ -2409,7 +2417,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/usermap', }, metadata: { - job_id: 7, + jobId: 7, userId: 'u1', }, destination: { @@ -2483,7 +2491,7 @@ export const data = [ }, metadata: [ { - job_id: 1, + jobId: 1, userId: 'u1', }, ], @@ -2519,7 +2527,7 @@ export const data = [ }, metadata: [ { - job_id: 6, + jobId: 6, userId: 'u1', }, ], @@ -2553,7 +2561,7 @@ export const data = [ }, metadata: [ { - job_id: 7, + jobId: 7, userId: 'u1', }, ], @@ -2664,19 +2672,19 @@ export const data = [ }, metadata: [ { - job_id: 2, + jobId: 2, userId: 'u1', }, { - job_id: 3, + jobId: 3, userId: 'u1', }, { - job_id: 4, + jobId: 4, userId: 'u1', }, { - job_id: 5, + jobId: 5, userId: 'u1', }, ], @@ -2756,7 +2764,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 1, + jobId: 1, userId: 'u1', }, destination: { @@ -2811,7 +2819,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 2, + jobId: 2, userId: 'u1', }, destination: { @@ -2866,7 +2874,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 3, + jobId: 3, userId: 'u1', }, destination: { @@ -2924,7 +2932,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 4, + jobId: 4, userId: 'u1', }, destination: { @@ -2982,7 +2990,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/2/httpapi', }, metadata: { - job_id: 5, + jobId: 5, userId: 'u1', }, destination: { @@ -3016,7 +3024,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/groupidentify', }, metadata: { - job_id: 6, + jobId: 6, userId: 'u1', }, destination: { @@ -3048,7 +3056,7 @@ export const data = [ endpoint: 'https://api2.amplitude.com/usermap', }, metadata: { - job_id: 7, + jobId: 7, userId: 'u1', }, destination: { @@ -3121,7 +3129,7 @@ export const data = [ }, metadata: [ { - job_id: 1, + jobId: 1, userId: 'u1', }, ], @@ -3157,7 +3165,7 @@ export const data = [ }, metadata: [ { - job_id: 6, + jobId: 6, userId: 'u1', }, ], @@ -3191,7 +3199,7 @@ export const data = [ }, metadata: [ { - job_id: 7, + jobId: 7, userId: 'u1', }, ], @@ -3302,19 +3310,19 @@ export const data = [ }, metadata: [ { - job_id: 2, + jobId: 2, userId: 'u1', }, { - job_id: 3, + jobId: 3, userId: 'u1', }, { - job_id: 4, + jobId: 4, userId: 'u1', }, { - job_id: 5, + jobId: 5, userId: 'u1', }, ], From a98cabdfe7781ada12baf742df4a3a439fc5fecd Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:01:41 +0530 Subject: [PATCH 09/13] fix: gaoc batching order (#3064) * fix: gaoc batching order * refactor: apply review suggestions --- .../transform.js | 3 +- src/v0/destinations/mp/transform.js | 2 +- src/v0/destinations/mp/util.js | 91 ------ src/v0/destinations/mp/util.test.js | 230 -------------- src/v0/util/index.js | 85 +++++ src/v0/util/index.test.js | 292 ++++++++++++++++- .../router/data.ts | 294 ++++++++---------- 7 files changed, 502 insertions(+), 495 deletions(-) diff --git a/src/v0/destinations/google_adwords_offline_conversions/transform.js b/src/v0/destinations/google_adwords_offline_conversions/transform.js index 397895c603..46cde72771 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/transform.js +++ b/src/v0/destinations/google_adwords_offline_conversions/transform.js @@ -10,6 +10,7 @@ const { defaultBatchRequestConfig, getSuccessRespEvents, checkInvalidRtTfEvents, + combineBatchRequestsWithSameJobIds, } = require('../../util'); const { CALL_CONVERSION, @@ -229,7 +230,7 @@ const processRouterDest = async (inputs, reqMetadata) => { .concat(storeSalesEventsBatchedResponseList) .concat(clickCallEvents) .concat(errorRespList); - return batchedResponseList; + return combineBatchRequestsWithSameJobIds(batchedResponseList); }; module.exports = { process, processRouterDest }; diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 41a814683d..24890c0eb1 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -18,6 +18,7 @@ const { handleRtTfSingleEventError, groupEventsByType, parseConfigArray, + combineBatchRequestsWithSameJobIds, } = require('../../util'); const { ConfigCategory, @@ -33,7 +34,6 @@ const { createIdentifyResponse, isImportAuthCredentialsAvailable, buildUtmParams, - combineBatchRequestsWithSameJobIds, groupEventsByEndpoint, batchEvents, trimTraits, diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index c01d2308b7..8e943f41dd 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -139,44 +139,6 @@ const isImportAuthCredentialsAvailable = (destination) => destination.Config.serviceAccountUserName && destination.Config.projectId); -/** - * Finds an existing batch based on metadata JobIds from the provided batch and metadataMap. - * @param {*} batch - * @param {*} metadataMap The map containing metadata items indexed by JobIds. - * @returns - */ -const findExistingBatch = (batch, metadataMap) => { - let existingBatch = null; - - // eslint-disable-next-line no-restricted-syntax - for (const metadataItem of batch.metadata) { - if (metadataMap.has(metadataItem.jobId)) { - existingBatch = metadataMap.get(metadataItem.jobId); - break; - } - } - - return existingBatch; -}; - -/** - * Removes duplicate metadata within each merged batch object. - * @param {*} mergedBatches An array of merged batch objects. - */ -const removeDuplicateMetadata = (mergedBatches) => { - mergedBatches.forEach((batch) => { - const metadataSet = new Set(); - // eslint-disable-next-line no-param-reassign - batch.metadata = batch.metadata.filter((metadataItem) => { - if (!metadataSet.has(metadataItem.jobId)) { - metadataSet.add(metadataItem.jobId); - return true; - } - return false; - }); - }); -}; - /** * Builds UTM parameters from a campaign object. * @@ -273,58 +235,6 @@ const batchEvents = (successRespList, maxBatchSize, reqMetadata) => { }); }; -/** - * Combines batched requests with the same JobIds. - * @param {*} inputBatches The array of batched request objects. - * @returns The combined batched requests with merged JobIds. - * - */ -const combineBatchRequestsWithSameJobIds = (inputBatches) => { - const combineBatches = (batches) => { - const clonedBatches = [...batches]; - const mergedBatches = []; - const metadataMap = new Map(); - - clonedBatches.forEach((batch) => { - const existingBatch = findExistingBatch(batch, metadataMap); - - if (existingBatch) { - // Merge batchedRequests arrays - existingBatch.batchedRequest = [ - ...(Array.isArray(existingBatch.batchedRequest) - ? existingBatch.batchedRequest - : [existingBatch.batchedRequest]), - ...(Array.isArray(batch.batchedRequest) ? batch.batchedRequest : [batch.batchedRequest]), - ]; - - // Merge metadata - batch.metadata.forEach((metadataItem) => { - if (!metadataMap.has(metadataItem.jobId)) { - metadataMap.set(metadataItem.jobId, existingBatch); - } - existingBatch.metadata.push(metadataItem); - }); - } else { - mergedBatches.push(batch); - batch.metadata.forEach((metadataItem) => { - metadataMap.set(metadataItem.jobId, batch); - }); - } - }); - - // Remove duplicate metadata within each merged object - removeDuplicateMetadata(mergedBatches); - - return mergedBatches; - }; - // We need to run this twice because in first pass some batches might not get merged - // and in second pass they might get merged - // Example: [[{jobID:1}, {jobID:2}], [{jobID:3}], [{jobID:1}, {jobID:3}]] - // 1st pass: [[{jobID:1}, {jobID:2}, {jobID:3}], [{jobID:3}]] - // 2nd pass: [[{jobID:1}, {jobID:2}, {jobID:3}]] - return combineBatches(combineBatches(inputBatches)); -}; - /** * Trims the traits and contextTraits objects based on the setOnceProperties array and returns an object containing the modified traits, contextTraits, and setOnce properties. * @@ -398,6 +308,5 @@ module.exports = { groupEventsByEndpoint, generateBatchedPayloadForArray, batchEvents, - combineBatchRequestsWithSameJobIds, trimTraits, }; diff --git a/src/v0/destinations/mp/util.test.js b/src/v0/destinations/mp/util.test.js index ebf140fadd..866119a336 100644 --- a/src/v0/destinations/mp/util.test.js +++ b/src/v0/destinations/mp/util.test.js @@ -1,5 +1,4 @@ const { - combineBatchRequestsWithSameJobIds, groupEventsByEndpoint, batchEvents, generateBatchedPayloadForArray, @@ -263,235 +262,6 @@ describe('Mixpanel utils test', () => { }); }); - describe('Unit test cases for combineBatchRequestsWithSameJobIds', () => { - it('Combine batch request with same jobIds', async () => { - const input = [ - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/track/', - }, - metadata: [ - { - jobId: 1, - }, - { - jobId: 4, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/import/', - }, - metadata: [ - { - jobId: 3, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/track/', - }, - metadata: [ - { - jobId: 5, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/engage/', - }, - metadata: [ - { - jobId: 1, - }, - { - jobId: 3, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/import/', - }, - metadata: [ - { - jobId: 6, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - ]; - - const expectedOutput = [ - { - batchedRequest: [ - { - endpoint: 'https://api.mixpanel.com/track/', - }, - { - endpoint: 'https://api.mixpanel.com/engage/', - }, - { - endpoint: 'https://api.mixpanel.com/import/', - }, - ], - metadata: [ - { - jobId: 1, - }, - { - jobId: 4, - }, - { - jobId: 3, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/track/', - }, - metadata: [ - { - jobId: 5, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/import/', - }, - metadata: [ - { - jobId: 6, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - ]; - expect(combineBatchRequestsWithSameJobIds(input)).toEqual(expectedOutput); - }); - - it('Each batchRequest contains unique jobIds (no event multiplexing)', async () => { - const input = [ - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/track/', - }, - metadata: [ - { - jobId: 1, - }, - { - jobId: 4, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/engage/', - }, - metadata: [ - { - jobId: 2, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/engage/', - }, - metadata: [ - { - jobId: 5, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - ]; - - const expectedOutput = [ - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/track/', - }, - - metadata: [ - { - jobId: 1, - }, - { - jobId: 4, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/engage/', - }, - metadata: [ - { - jobId: 2, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - { - batchedRequest: { - endpoint: 'https://api.mixpanel.com/engage/', - }, - metadata: [ - { - jobId: 5, - }, - ], - batched: true, - statusCode: 200, - destination: destinationMock, - }, - ]; - expect(combineBatchRequestsWithSameJobIds(input)).toEqual(expectedOutput); - }); - }); - describe('Unit test cases for generateBatchedPayloadForArray', () => { it('should generate a batched payload with GZIP payload for /import endpoint when given an array of events', () => { const events = [ diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 49ef39969e..0cc66b2d7a 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -33,6 +33,7 @@ const { AUTH_STATUS_INACTIVE, } = require('../../adapters/networkhandler/authConstants'); const { FEATURE_FILTER_CODE, FEATURE_GZIP_SUPPORT } = require('./constant'); +const { CommonUtils } = require('../../util/common'); // ======================================================================== // INLINERS @@ -2136,6 +2137,87 @@ const parseConfigArray = (arr, key) => { return arr.map((item) => item[key]); }; +/** + * Finds an existing batch based on metadata JobIds from the provided batch and metadataMap. + * @param {*} batch + * @param {*} metadataMap The map containing metadata items indexed by JobIds. + * @returns + */ +const findExistingBatch = (batch, metadataMap) => { + const existingMetadataItem = batch.metadata.find((metadataItem) => + metadataMap.has(metadataItem.jobId), + ); + return existingMetadataItem ? metadataMap.get(existingMetadataItem.jobId) : null; +}; + +/** + * Removes duplicate metadata within each merged batch object. + * @param {*} mergedBatches An array of merged batch objects. + */ +const removeDuplicateMetadata = (mergedBatches) => { + mergedBatches.forEach((batch) => { + const metadataSet = new Set(); + // eslint-disable-next-line no-param-reassign + batch.metadata = batch.metadata.filter((metadataItem) => { + if (!metadataSet.has(metadataItem.jobId)) { + metadataSet.add(metadataItem.jobId); + return true; + } + return false; + }); + }); +}; + +/** + * Combines batched requests with the same JobIds. + * @param {*} inputBatches The array of batched request objects. + * @returns The combined batched requests with merged JobIds. + * + */ +const combineBatchRequestsWithSameJobIds = (inputBatches) => { + const combineBatches = (batches) => { + const clonedBatches = [...batches]; + const mergedBatches = []; + const metadataMap = new Map(); + + clonedBatches.forEach((batch) => { + const existingBatch = findExistingBatch(batch, metadataMap); + + if (existingBatch) { + // Merge batchedRequests arrays + existingBatch.batchedRequest = [ + ...CommonUtils.toArray(existingBatch.batchedRequest), + ...CommonUtils.toArray(batch.batchedRequest), + ]; + + // Merge metadata + batch.metadata.forEach((metadataItem) => { + if (!metadataMap.has(metadataItem.jobId)) { + metadataMap.set(metadataItem.jobId, existingBatch); + } + existingBatch.metadata.push(metadataItem); + }); + } else { + mergedBatches.push(batch); + batch.metadata.forEach((metadataItem) => { + metadataMap.set(metadataItem.jobId, batch); + }); + } + }); + + // Remove duplicate metadata within each merged object + removeDuplicateMetadata(mergedBatches); + + return mergedBatches; + }; + // We need to run this twice because in first pass some batches might not get merged + // and in second pass they might get merged + // Example: [[{jobID:1}, {jobID:2}], [{jobID:3}], [{jobID:1}, {jobID:3}]] + // 1st pass: [[{jobID:1}, {jobID:2}, {jobID:3}], [{jobID:3}]] + // 2nd pass: [[{jobID:1}, {jobID:2}, {jobID:3}]] + return combineBatches(combineBatches(inputBatches)); +}; + // ======================================================================== // EXPORTS // ======================================================================== @@ -2249,4 +2331,7 @@ module.exports = { isNewStatusCodesAccepted, IsGzipSupported, parseConfigArray, + findExistingBatch, + removeDuplicateMetadata, + combineBatchRequestsWithSameJobIds, }; diff --git a/src/v0/util/index.test.js b/src/v0/util/index.test.js index 1c6b34eca6..4dc6255691 100644 --- a/src/v0/util/index.test.js +++ b/src/v0/util/index.test.js @@ -2,7 +2,12 @@ const { TAG_NAMES } = require('@rudderstack/integrations-lib'); const utilities = require('.'); const { getFuncTestData } = require('../../../test/testHelper'); const { FilteredEventsError } = require('./errorTypes'); -const { hasCircularReference, flattenJson, generateExclusionList } = require('./index'); +const { + hasCircularReference, + flattenJson, + generateExclusionList, + combineBatchRequestsWithSameJobIds, +} = require('./index'); // Names of the utility functions to test const functionNames = [ @@ -166,3 +171,288 @@ describe('generateExclusionList', () => { expect(result).toEqual(expected); }); }); + +describe('Unit test cases for combineBatchRequestsWithSameJobIds', () => { + it('Combine batch request with same jobIds', async () => { + const input = [ + { + batchedRequest: { + endpoint: 'https://endpoint1', + }, + metadata: [ + { + jobId: 1, + }, + { + jobId: 4, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + { + batchedRequest: { + endpoint: 'https://endpoint2', + }, + metadata: [ + { + jobId: 3, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + { + batchedRequest: { + endpoint: 'https://endpoint1', + }, + metadata: [ + { + jobId: 5, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + { + batchedRequest: { + endpoint: 'https://endpoint3', + }, + metadata: [ + { + jobId: 1, + }, + { + jobId: 3, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + { + batchedRequest: { + endpoint: 'https://endpoint2', + }, + metadata: [ + { + jobId: 6, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + ]; + + const expectedOutput = [ + { + batchedRequest: [ + { + endpoint: 'https://endpoint1', + }, + { + endpoint: 'https://endpoint3', + }, + { + endpoint: 'https://endpoint2', + }, + ], + metadata: [ + { + jobId: 1, + }, + { + jobId: 4, + }, + { + jobId: 3, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + { + batchedRequest: { + endpoint: 'https://endpoint1', + }, + metadata: [ + { + jobId: 5, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + { + batchedRequest: { + endpoint: 'https://endpoint2', + }, + metadata: [ + { + jobId: 6, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + ]; + expect(combineBatchRequestsWithSameJobIds(input)).toEqual(expectedOutput); + }); + + it('Each batchRequest contains unique jobIds (no event multiplexing)', async () => { + const input = [ + { + batchedRequest: { + endpoint: 'https://endpoint1', + }, + metadata: [ + { + jobId: 1, + }, + { + jobId: 4, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + { + batchedRequest: { + endpoint: 'https://endpoint3', + }, + metadata: [ + { + jobId: 2, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + { + batchedRequest: { + endpoint: 'https://endpoint3', + }, + metadata: [ + { + jobId: 5, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + ]; + + const expectedOutput = [ + { + batchedRequest: { + endpoint: 'https://endpoint1', + }, + + metadata: [ + { + jobId: 1, + }, + { + jobId: 4, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + { + batchedRequest: { + endpoint: 'https://endpoint3', + }, + metadata: [ + { + jobId: 2, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + { + batchedRequest: { + endpoint: 'https://endpoint3', + }, + metadata: [ + { + jobId: 5, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + key: 'value', + }, + }, + }, + ]; + expect(combineBatchRequestsWithSameJobIds(input)).toEqual(expectedOutput); + }); +}); diff --git a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts index 1536af8187..a38980f0e9 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts @@ -96,6 +96,7 @@ export const data = [ refresh_token: 'efgh5678', developer_token: 'ijkl91011', }, + jobId: 1, userId: 'u1', }, destination: { @@ -211,6 +212,7 @@ export const data = [ refresh_token: 'efgh5678', developer_token: 'ijkl91011', }, + jobId: 2, userId: 'u1', }, destination: { @@ -274,7 +276,7 @@ export const data = [ refresh_token: 'efgh5678', developer_token: 'ijkl91011', }, - jobId: 1, + jobId: 3, userId: 'u1', }, destination: { @@ -350,7 +352,7 @@ export const data = [ refresh_token: 'efgh5678', developer_token: 'ijkl91011', }, - jobId: 2, + jobId: 4, userId: 'u1', }, destination: { @@ -426,7 +428,7 @@ export const data = [ refresh_token: 'efgh5678', developer_token: 'ijkl91011', }, - jobId: 3, + jobId: 5, userId: 'u1', }, destination: { @@ -479,80 +481,130 @@ export const data = [ body: { output: [ { - batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: - 'https://googleads.googleapis.com/v14/customers/7693729833/offlineUserDataJobs', - headers: { - Authorization: 'Bearer abcd1234', - 'Content-Type': 'application/json', - 'developer-token': 'ijkl91011', - }, - params: { event: 'Store sales', customerId: '7693729833' }, - body: { - JSON: { - event: '7693729833', - isStoreConversion: true, - createJobPayload: { - job: { - storeSalesMetadata: { - loyaltyFraction: 1, - transaction_upload_fraction: '1', + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://googleads.googleapis.com/v14/customers/7693729833/offlineUserDataJobs', + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + }, + params: { event: 'Store sales', customerId: '7693729833' }, + body: { + JSON: { + event: '7693729833', + isStoreConversion: true, + createJobPayload: { + job: { + storeSalesMetadata: { + loyaltyFraction: 1, + transaction_upload_fraction: '1', + }, + type: 'STORE_SALES_UPLOAD_FIRST_PARTY', }, - type: 'STORE_SALES_UPLOAD_FIRST_PARTY', }, - }, - addConversionPayload: { - operations: [ - { - create: { - transaction_attribute: { - store_attribute: { store_code: 'store code' }, - transaction_amount_micros: '100000000', - order_id: 'order id', - currency_code: 'INR', - transaction_date_time: '2019-10-14 16:45:18+05:30', - }, - userIdentifiers: [ - { - hashedEmail: - '6db61e6dcbcf2390e4a46af426f26a133a3bee45021422fc7ae86e9136f14110', + addConversionPayload: { + operations: [ + { + create: { + transaction_attribute: { + store_attribute: { store_code: 'store code' }, + transaction_amount_micros: '100000000', + order_id: 'order id', + currency_code: 'INR', + transaction_date_time: '2019-10-14 16:45:18+05:30', }, - ], - }, - }, - { - create: { - transaction_attribute: { - store_attribute: { store_code: 'store code2' }, - transaction_amount_micros: '100000000', - order_id: 'order id', - currency_code: 'INR', - transaction_date_time: '2019-10-14 16:45:18+05:30', + userIdentifiers: [ + { + hashedEmail: + '6db61e6dcbcf2390e4a46af426f26a133a3bee45021422fc7ae86e9136f14110', + }, + ], }, - userIdentifiers: [ - { - hashedEmail: - '6db61e6dcbcf2390e4a46af426f26a133a3bee45021422fc7ae86e9136f14110', + }, + { + create: { + transaction_attribute: { + store_attribute: { store_code: 'store code2' }, + transaction_amount_micros: '100000000', + order_id: 'order id', + currency_code: 'INR', + transaction_date_time: '2019-10-14 16:45:18+05:30', }, - ], + userIdentifiers: [ + { + hashedEmail: + '6db61e6dcbcf2390e4a46af426f26a133a3bee45021422fc7ae86e9136f14110', + }, + ], + }, }, + ], + enable_partial_failure: false, + enable_warnings: false, + validate_only: true, + }, + executeJobPayload: { validate_only: true }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://googleads.googleapis.com/v14/customers/7693729833:uploadCallConversions', + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + }, + params: { + event: 'Order Completed', + customerId: '7693729833', + customVariables: [{ from: '', to: '' }], + properties: { + loyaltyFraction: 1, + order_id: 'order id', + currency: 'INR', + revenue: '100', + store_code: 'store code2', + email: 'alex@example.com', + gclid: 'gclid', + product_id: '123445', + quantity: 123, + callerId: '1234', + callStartDateTime: '2019-10-14T11:15:18.299Z', + }, + }, + body: { + JSON: { + conversions: [ + { + callerId: '1234', + callStartDateTime: '2019-10-14T11:15:18.299Z', + conversionDateTime: '2019-10-14 16:45:18+05:30', + conversionValue: 100, + currencyCode: 'INR', }, ], - enable_partial_failure: false, - enable_warnings: false, - validate_only: true, + partialFailure: true, }, - executeJobPayload: { validate_only: true }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, + files: {}, }, - files: {}, - }, + ], metadata: [ { secret: { @@ -560,7 +612,7 @@ export const data = [ refresh_token: 'efgh5678', developer_token: 'ijkl91011', }, - jobId: 1, + jobId: 3, userId: 'u1', }, { @@ -569,7 +621,7 @@ export const data = [ refresh_token: 'efgh5678', developer_token: 'ijkl91011', }, - jobId: 2, + jobId: 4, userId: 'u1', }, ], @@ -714,6 +766,7 @@ export const data = [ refresh_token: 'efgh5678', developer_token: 'ijkl91011', }, + jobId: 1, userId: 'u1', }, ], @@ -824,6 +877,7 @@ export const data = [ refresh_token: 'efgh5678', developer_token: 'ijkl91011', }, + jobId: 2, userId: 'u1', }, ], @@ -857,108 +911,6 @@ export const data = [ }, }, }, - { - batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: - 'https://googleads.googleapis.com/v14/customers/7693729833:uploadCallConversions', - headers: { - Authorization: 'Bearer abcd1234', - 'Content-Type': 'application/json', - 'developer-token': 'ijkl91011', - }, - params: { - event: 'Order Completed', - customerId: '7693729833', - customVariables: [{ from: '', to: '' }], - properties: { - loyaltyFraction: 1, - order_id: 'order id', - currency: 'INR', - revenue: '100', - store_code: 'store code2', - email: 'alex@example.com', - gclid: 'gclid', - product_id: '123445', - quantity: 123, - callerId: '1234', - callStartDateTime: '2019-10-14T11:15:18.299Z', - }, - }, - body: { - JSON: { - conversions: [ - { - callerId: '1234', - callStartDateTime: '2019-10-14T11:15:18.299Z', - conversionDateTime: '2019-10-14 16:45:18+05:30', - conversionValue: 100, - currencyCode: 'INR', - }, - ], - partialFailure: true, - }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, - }, - files: {}, - }, - metadata: [ - { - secret: { - access_token: 'abcd1234', - refresh_token: 'efgh5678', - developer_token: 'ijkl91011', - }, - jobId: 2, - userId: 'u1', - }, - ], - batched: false, - statusCode: 200, - destination: { - Config: { - rudderAccountId: '2Hsy2iFyoG5VLDd9wQcggHLMYFA', - customerId: '769-372-9833', - subAccount: false, - UserIdentifierSource: 'FIRST_PARTY', - conversionEnvironment: 'none', - defaultUserIdentifier: 'email', - hashUserIdentifier: true, - validateOnly: true, - eventsToOfflineConversionsTypeMapping: [ - { from: 'Data Reading Guide', to: 'click' }, - { from: 'Order Completed', to: 'store' }, - { from: 'Sign-up - click', to: 'click' }, - { from: 'Outbound click (rudderstack.com)', to: 'click' }, - { from: 'Page view', to: 'click' }, - { from: 'download', to: 'click' }, - { from: 'Product Clicked', to: 'store' }, - { from: 'Order Completed', to: 'call' }, - ], - loginCustomerId: '4219454086', - eventsToConversionsNamesMapping: [ - { from: 'Data Reading Guide', to: 'Data Reading Guide' }, - { from: 'Order Completed', to: 'Order Completed' }, - { from: 'Sign-up - click', to: 'Sign-up - click' }, - { - from: 'Outbound click (rudderstack.com)', - to: 'Outbound click (rudderstack.com)', - }, - { from: 'Page view', to: 'Page view' }, - { from: 'Sign up completed', to: 'Sign-up - click' }, - { from: 'download', to: 'Page view' }, - { from: 'Product Clicked', to: 'Store sales' }, - ], - authStatus: 'active', - oneTrustCookieCategories: [], - customVariables: [{ from: '', to: '' }], - }, - }, - }, { metadata: [ { @@ -967,7 +919,7 @@ export const data = [ refresh_token: 'efgh5678', developer_token: 'ijkl91011', }, - jobId: 3, + jobId: 5, userId: 'u1', }, ], From d522b35c908a9f262ba3ba27dda0ea5d9ac5bc6b Mon Sep 17 00:00:00 2001 From: Sandeep Digumarty Date: Tue, 13 Feb 2024 16:41:12 +0530 Subject: [PATCH 10/13] fix: resolve bugsnag issue caused due to undefined properties (#3086) --- src/v0/util/facebookUtils/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v0/util/facebookUtils/index.js b/src/v0/util/facebookUtils/index.js index 4c09518559..7fa1e898fe 100644 --- a/src/v0/util/facebookUtils/index.js +++ b/src/v0/util/facebookUtils/index.js @@ -147,7 +147,7 @@ const getContentType = (message, defaultValue, categoryToContent, destinationNam return integrationsObj.contentType; } - let { category } = properties; + let { category } = properties || {}; if (!category) { const { products } = properties; if (products && products.length > 0 && Array.isArray(products) && isObject(products[0])) { From f7ec0a1244a7b97e6b40de5ed9881c63300866dc Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Wed, 14 Feb 2024 10:42:29 +0530 Subject: [PATCH 11/13] fix: amplitude: Error handling for missing event type (#3079) * fix: amplitude: Error handling for missing event type * chore: small fix for undefined userDefinedTemplate * Update src/v0/destinations/am/transform.js Co-authored-by: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> * chore: comments addressed * fix: test cases --------- Co-authored-by: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> --- src/v0/destinations/am/transform.js | 7 +-- src/v0/destinations/am/util.test.js | 33 +++++++++++- src/v0/destinations/am/utils.js | 15 ++++++ .../destinations/am/processor/data.ts | 53 +++++++++++++++++++ 4 files changed, 104 insertions(+), 4 deletions(-) diff --git a/src/v0/destinations/am/transform.js b/src/v0/destinations/am/transform.js index d78a5f727f..afd72b77e1 100644 --- a/src/v0/destinations/am/transform.js +++ b/src/v0/destinations/am/transform.js @@ -614,16 +614,16 @@ const processSingleMessage = (message, destination) => { case EventType.PAGE: if (useUserDefinedPageEventName) { const getMessagePath = userProvidedPageEventString - .substring( + ?.substring( userProvidedPageEventString.indexOf('{') + 2, userProvidedPageEventString.indexOf('}'), ) .trim(); evType = - userProvidedPageEventString.trim() === '' + userProvidedPageEventString?.trim() === '' ? name : userProvidedPageEventString - .trim() + ?.trim() .replaceAll(/{{([^{}]+)}}/g, get(message, getMessagePath)); } else { evType = `Viewed ${name || get(message, CATEGORY_KEY) || ''} Page`; @@ -701,6 +701,7 @@ const processSingleMessage = (message, destination) => { logger.debug('could not determine type'); throw new InstrumentationError('message type not supported'); } + AMUtils.validateEventType(evType); return responseBuilderSimple( groupInfo, payloadObjectName, diff --git a/src/v0/destinations/am/util.test.js b/src/v0/destinations/am/util.test.js index faaa9170f0..455f9117ef 100644 --- a/src/v0/destinations/am/util.test.js +++ b/src/v0/destinations/am/util.test.js @@ -1,4 +1,4 @@ -const { getUnsetObj } = require('./utils'); +const { getUnsetObj, validateEventType } = require('./utils'); describe('getUnsetObj', () => { it("should return undefined when 'message.integrations.Amplitude.fieldsToUnset' is not array", () => { @@ -64,3 +64,34 @@ describe('getUnsetObj', () => { expect(result).toBeUndefined(); }); }); + + +describe('validateEventType', () => { + + it('should validate event type when it is valid with only page name given', () => { + expect(() => { + validateEventType('Home Page'); + }).not.toThrow(); + }); + + it('should throw an error when event type is null', () => { + expect(() => { + validateEventType(null); + }).toThrow('Event type is missing. Please send it under `event.type`. For page/screen events, send it under `event.name`'); + }); + + it('should throw an error when event type is undefined', () => { + expect(() => { + validateEventType(undefined); + }).toThrow('Event type is missing. Please send it under `event.type`. For page/screen events, send it under `event.name`'); + }); + + // Function receives an empty string as event type + it('should throw an error when event type is an empty string', () => { + expect(() => { + validateEventType(''); + }).toThrow('Event type is missing. Please send it under `event.type`. For page/screen events, send it under `event.name`'); + }); + +}); + diff --git a/src/v0/destinations/am/utils.js b/src/v0/destinations/am/utils.js index 71fe0ab459..e51d09aaa7 100644 --- a/src/v0/destinations/am/utils.js +++ b/src/v0/destinations/am/utils.js @@ -10,6 +10,7 @@ // populate these dest keys const get = require('get-value'); const uaParser = require('@amplitude/ua-parser-js'); +const { InstrumentationError } = require('@rudderstack/integrations-lib'); const logger = require('../../../logger'); const { isDefinedAndNotNull } = require('../../util'); @@ -108,6 +109,19 @@ const getUnsetObj = (message) => { return unsetObject; }; + +/** + * Check for evType as in some cases, like when the page name is absent, + * either the template depends only on the event.name or there is no template provided by user + * @param {*} evType + */ +const validateEventType = (evType) => { + if (!isDefinedAndNotNull(evType) || (typeof evType === "string" && evType.length ===0)) { + throw new InstrumentationError( + 'Event type is missing. Please send it under `event.type`. For page/screen events, send it under `event.name`', + ); + } +}; module.exports = { getOSName, getOSVersion, @@ -117,4 +131,5 @@ module.exports = { getBrand, getEventId, getUnsetObj, + validateEventType }; diff --git a/test/integrations/destinations/am/processor/data.ts b/test/integrations/destinations/am/processor/data.ts index f28606da0c..b645fb5ac7 100644 --- a/test/integrations/destinations/am/processor/data.ts +++ b/test/integrations/destinations/am/processor/data.ts @@ -11327,4 +11327,57 @@ export const data = [ }, }, }, + { + name: 'am', + description: + 'Test 78 -> Page call invalid event type as page name and template is not provided', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + request_ip: '1.1.1.1', + type: 'page', + userId: '12345', + properties: {}, + integrations: { + All: true, + }, + sentAt: '2019-10-14T11:15:53.296Z', + }, + destination: { + Config: { + apiKey: 'abcde', + useUserDefinedPageEventName: true, + userProvidedPageEventString: '', + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 400, + error: + 'Event type is missing. Please send it under `event.type`. For page/screen events, send it under `event.name`', + statTags: { + errorCategory: 'dataValidation', + errorType: 'instrumentation', + destType: 'AM', + module: 'destination', + implementation: 'native', + feature: 'processor', + }, + }, + ], + }, + }, + }, ]; From b6edff46fa0e0e210e82206fea46a064e3fbe00f Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Wed, 14 Feb 2024 13:57:10 +0530 Subject: [PATCH 12/13] fix: tiktok ads v2 error handling (#3084) * fix: tiktok ads v2 error handling * refactor: remove redundant import --- src/v0/destinations/tiktok_ads/transformV2.js | 2 +- .../destinations/tiktok_ads/processor/data.ts | 103 ++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/src/v0/destinations/tiktok_ads/transformV2.js b/src/v0/destinations/tiktok_ads/transformV2.js index 91078dfe65..98f7d61e1e 100644 --- a/src/v0/destinations/tiktok_ads/transformV2.js +++ b/src/v0/destinations/tiktok_ads/transformV2.js @@ -40,7 +40,7 @@ const getTrackResponsePayload = (message, destConfig, event) => { } // if contents is not present but we have properties.products present which has fields with superset of contents fields - if (payload.properties && !payload.properties.contents && message.properties.products) { + if (!payload.properties?.contents && message.properties?.products) { // retreiving data from products only when contents is not present payload.properties.contents = getContents(message, false); } diff --git a/test/integrations/destinations/tiktok_ads/processor/data.ts b/test/integrations/destinations/tiktok_ads/processor/data.ts index 9d7c3a8d10..3b68426fbf 100644 --- a/test/integrations/destinations/tiktok_ads/processor/data.ts +++ b/test/integrations/destinations/tiktok_ads/processor/data.ts @@ -6870,4 +6870,107 @@ export const data = [ }, }, }, + { + name: 'tiktok_ads', + description: 'Test 46 -> V2 -> Event with no properties', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + anonymousId: '21e13f4bc7ceddad', + channel: 'web', + context: { + traits: { + email: 'dd6ff77f54e2106661089bae4d40cdb600979bf7edc9eb65c0942ba55c7c2d7f', + }, + userAgent: + 'Mozilla/5.0 (platform; rv:geckoversion) Gecko/geckotrail Firefox/firefoxversion', + ip: '13.57.97.131', + locale: 'en-US', + externalId: [ + { + type: 'tiktokExternalId', + id: 'f0e388f53921a51f0bb0fc8a2944109ec188b59172935d8f23020b1614cc44bc', + }, + ], + }, + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + timestamp: '2020-09-17T19:49:27Z', + type: 'track', + event: 'customEvent', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + Config: { + version: 'v2', + accessToken: 'dummyAccessToken', + pixelCode: '{{PIXEL-CODE}}', + hashUserProperties: false, + sendCustomEvents: true, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://business-api.tiktok.com/open_api/v1.3/event/track/', + headers: { + 'Access-Token': 'dummyAccessToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + event_source: 'web', + event_source_id: '{{PIXEL-CODE}}', + partner_name: 'RudderStack', + data: [ + { + event: 'customEvent', + event_id: '84e26acc-56a5-4835-8233-591137fca468', + event_time: 1600372167, + properties: { content_type: 'product' }, + user: { + locale: 'en-US', + email: 'dd6ff77f54e2106661089bae4d40cdb600979bf7edc9eb65c0942ba55c7c2d7f', + external_id: + 'f0e388f53921a51f0bb0fc8a2944109ec188b59172935d8f23020b1614cc44bc', + ip: '13.57.97.131', + user_agent: + 'Mozilla/5.0 (platform; rv:geckoversion) Gecko/geckotrail Firefox/firefoxversion', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, ]; From 6a364fba34c46b15c0fe4b06ecfa6f4b81b6f436 Mon Sep 17 00:00:00 2001 From: Mihir Bhalala <77438541+mihir-4116@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:39:25 +0530 Subject: [PATCH 13/13] fix(ga4): failures not considered with 200 status in events tab (#3089) --- src/v0/destinations/ga4/networkHandler.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/ga4/networkHandler.js b/src/v0/destinations/ga4/networkHandler.js index b62fcc8d3b..2cb98e1460 100644 --- a/src/v0/destinations/ga4/networkHandler.js +++ b/src/v0/destinations/ga4/networkHandler.js @@ -30,9 +30,9 @@ const responseHandler = (destinationResponse, dest) => { const { description, validationCode, fieldPath } = response.validationMessages[0]; throw new NetworkError( `Validation Server Response Handler:: Validation Error for ${dest} of field path :${fieldPath} | ${validationCode}-${description}`, - status, + 400, { - [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(400), }, response?.validationMessages[0]?.description, );