From 1c8e950f3d8789b33bba69a30c9eb21c40ce3d04 Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Tue, 25 Jun 2024 12:19:19 +0530 Subject: [PATCH 1/5] fix: enhancement: introduce user model for one signal (#3499) * feat: onboard user API for onesignal * chore: add aliases from one signal intg object * chore: add test cases * chore: add test cases+1 * chore: add test cases+2 * chore: small fixes * fix: lint errors * fix: lint errors+1 * fix: lint errors+2 * chore: rename file * chore: address comments --- src/v0/destinations/one_signal/config.js | 21 ++ .../data/OneSignalIdentifyConfigV2.json | 54 +++ .../data/OneSignalSubscriptionConfig.json | 8 + src/v0/destinations/one_signal/transform.js | 9 +- src/v0/destinations/one_signal/transformV2.js | 159 ++++++++ src/v0/destinations/one_signal/util.js | 158 +++++++- src/v0/destinations/one_signal/utils.test.js | 35 ++ .../one_signal/processor/commonConfig.ts | 45 +++ .../destinations/one_signal/processor/data.ts | 2 + .../one_signal/processor/data_v2.ts | 5 + .../one_signal/processor/group.ts | 121 +++++++ .../one_signal/processor/identify.ts | 341 ++++++++++++++++++ .../one_signal/processor/track.ts | 249 +++++++++++++ .../one_signal/processor/validations.ts | 149 ++++++++ 14 files changed, 1351 insertions(+), 5 deletions(-) create mode 100644 src/v0/destinations/one_signal/data/OneSignalIdentifyConfigV2.json create mode 100644 src/v0/destinations/one_signal/data/OneSignalSubscriptionConfig.json create mode 100644 src/v0/destinations/one_signal/transformV2.js create mode 100644 src/v0/destinations/one_signal/utils.test.js create mode 100644 test/integrations/destinations/one_signal/processor/commonConfig.ts create mode 100644 test/integrations/destinations/one_signal/processor/data_v2.ts create mode 100644 test/integrations/destinations/one_signal/processor/group.ts create mode 100644 test/integrations/destinations/one_signal/processor/identify.ts create mode 100644 test/integrations/destinations/one_signal/processor/track.ts create mode 100644 test/integrations/destinations/one_signal/processor/validations.ts diff --git a/src/v0/destinations/one_signal/config.js b/src/v0/destinations/one_signal/config.js index 1a58f3f91f..fdaa3ecd7f 100644 --- a/src/v0/destinations/one_signal/config.js +++ b/src/v0/destinations/one_signal/config.js @@ -1,6 +1,7 @@ const { getMappingConfig } = require('../../util'); const BASE_URL = 'https://onesignal.com/api/v1'; +const BASE_URL_V2 = 'https://api.onesignal.com/apps/{{app_id}}/users'; const ENDPOINTS = { IDENTIFY: { @@ -16,13 +17,33 @@ const ENDPOINTS = { const ConfigCategory = { IDENTIFY: { name: 'OneSignalIdentifyConfig', endpoint: '/players' }, + IDENTIFY_V2: { name: 'OneSignalIdentifyConfigV2' }, + SUBSCRIPTION: { name: 'OneSignalSubscriptionConfig' }, }; const mappingConfig = getMappingConfig(ConfigCategory, __dirname); +// Used for User Model (V2) +const deviceTypesV2Enums = [ + 'iOSPush', + 'email', + 'sms', + 'AndroidPush', + 'HuaweiPush', + 'FireOSPush', + 'WindowsPush', + 'macOSPush', + 'ChromeExtensionPush', + 'ChromePush', + 'SafariLegacyPush', + 'FirefoxPush', + 'SafariPush', +]; module.exports = { BASE_URL, + BASE_URL_V2, ENDPOINTS, ConfigCategory, mappingConfig, + deviceTypesV2Enums, }; diff --git a/src/v0/destinations/one_signal/data/OneSignalIdentifyConfigV2.json b/src/v0/destinations/one_signal/data/OneSignalIdentifyConfigV2.json new file mode 100644 index 0000000000..61ab6e0109 --- /dev/null +++ b/src/v0/destinations/one_signal/data/OneSignalIdentifyConfigV2.json @@ -0,0 +1,54 @@ +[ + { "sourceKeys": "context.locale", "destKey": "properties.laguage", "required": false }, + { "sourceKeys": "context.ip", "destKey": "properties.ip", "required": false }, + { "sourceKeys": "context.timezone", "destKey": "properties.timezone_id", "required": false }, + { "sourceKeys": "context.location.latitude", "destKey": "properties.lat", "required": false }, + { "sourceKeys": "context.location.longitude", "destKey": "properties.long", "required": false }, + { + "sourceKeys": "createdAt", + "destKey": "properties.created_at", + "sourceFromGenericMap": true, + "metadata": { + "type": "secondTimestamp" + }, + "required": false + }, + { + "sourceKeys": "createdAt", + "destKey": "properties.last_active", + "sourceFromGenericMap": true, + "metadata": { + "type": "secondTimestamp" + }, + "required": false + }, + { + "sourceKeys": [ + "traits.country", + "context.traits.country", + "traits.address.country", + "context.traits.address.country" + ], + "destKey": "properties.country", + "required": false + }, + { + "sourceKeys": [ + "traits.firstActive", + "context.traits.firstActive", + "traits.first_active", + "context.traits.first_active" + ], + "metadata": { + "type": "secondTimestamp" + }, + "destKey": "properties.first_active", + "required": false + }, + { + "sourceKeys": "userIdOnly", + "destKey": "identity.external_id", + "sourceFromGenericMap": true, + "required": false + } +] diff --git a/src/v0/destinations/one_signal/data/OneSignalSubscriptionConfig.json b/src/v0/destinations/one_signal/data/OneSignalSubscriptionConfig.json new file mode 100644 index 0000000000..4a7a877daa --- /dev/null +++ b/src/v0/destinations/one_signal/data/OneSignalSubscriptionConfig.json @@ -0,0 +1,8 @@ +[ + { "sourceKeys": "enabled", "destKey": "enabled", "required": false }, + { "sourceKeys": "notification_types", "destKey": "notification_types", "required": false }, + { "sourceKeys": "session_time", "destKey": "session_time", "required": false }, + { "sourceKeys": "session_count", "destKey": "session_count", "required": false }, + { "sourceKeys": "app_version", "destKey": "app_version", "required": false }, + { "sourceKeys": "test_type", "destKey": "test_type", "required": false } +] diff --git a/src/v0/destinations/one_signal/transform.js b/src/v0/destinations/one_signal/transform.js index b025660fa4..aac48e3b4e 100644 --- a/src/v0/destinations/one_signal/transform.js +++ b/src/v0/destinations/one_signal/transform.js @@ -4,6 +4,7 @@ const { TransformationError, InstrumentationError, } = require('@rudderstack/integrations-lib'); +const { process: processV2 } = require('./transformV2'); const { EventType } = require('../../../constants'); const { ConfigCategory, mappingConfig, BASE_URL, ENDPOINTS } = require('./config'); const { @@ -186,10 +187,16 @@ const groupResponseBuilder = (message, { Config }) => { }; const processEvent = (message, destination) => { + const { Config } = destination; + const { version, appId } = Config; + if (version === 'V2') { + // This version is used to direct the request to user centric model + return processV2(message, destination); + } if (!message.type) { throw new InstrumentationError('Event type is required'); } - if (!destination.Config.appId) { + if (!appId) { throw new ConfigurationError('appId is a required field'); } const messageType = message.type.toLowerCase(); diff --git a/src/v0/destinations/one_signal/transformV2.js b/src/v0/destinations/one_signal/transformV2.js new file mode 100644 index 0000000000..3d084e3c8a --- /dev/null +++ b/src/v0/destinations/one_signal/transformV2.js @@ -0,0 +1,159 @@ +const get = require('get-value'); +const { + ConfigurationError, + TransformationError, + InstrumentationError, +} = require('@rudderstack/integrations-lib'); +const { EventType } = require('../../../constants'); +const { ConfigCategory, mappingConfig, BASE_URL_V2 } = require('./config'); +const { + defaultRequestConfig, + getFieldValueFromMessage, + constructPayload, + defaultPostRequestConfig, + removeUndefinedAndNullValues, +} = require('../../util'); +const { + populateTags, + getProductPurchasesDetails, + getSubscriptions, + getOneSignalAliases, +} = require('./util'); +const { JSON_MIME_TYPE } = require('../../util/constant'); + +const responseBuilder = (payload, Config) => { + const { appId } = Config; + if (payload) { + const response = defaultRequestConfig(); + response.endpoint = `${BASE_URL_V2.replace('{{app_id}}', appId)}`; + response.headers = { + Accept: JSON_MIME_TYPE, + 'Content-Type': JSON_MIME_TYPE, + }; + response.method = defaultPostRequestConfig.requestMethod; + response.body.JSON = removeUndefinedAndNullValues(payload); + return response; + } + throw new TransformationError('Payload could not be populated due to wrong input'); +}; + +/** + * This function is used for creating response for identify call, to create a new user or update an existing user. + * a responseArray for creating/updating user is being prepared. + * If the value of emailDeviceType/smsDeviceType(toggle in dashboard) is true, separate responses will also be created + * for new subscriptions to be added to user with email/sms as token. + * @param {*} message + * @param {*} param1 + * @returns + */ +const identifyResponseBuilder = (message, { Config }) => { + // Populating the tags + const tags = populateTags(message); + + const payload = constructPayload(message, mappingConfig[ConfigCategory.IDENTIFY_V2.name]); + if (!payload?.identity?.external_id) { + const alias = getOneSignalAliases(message); + if (Object.keys(alias).length === 0) { + throw new InstrumentationError('userId or any other alias is required for identify'); + } + payload.identity = alias; + } + // Following check is to intialise properties object in case we don't get properties object from construct payload + if (!payload.properties) { + payload.properties = {}; + } + payload.subscriptions = getSubscriptions(message, Config); + payload.properties.tags = tags; + return responseBuilder(removeUndefinedAndNullValues(payload), Config); +}; + +/** + * This function is used to build the response for track call and Group call. + * It is used to edit the OneSignal tags using external_id. + * It edits tags[event] as true for track call + * @param {*} message + * @param {*} param1 + * @returns + */ +const trackOrGroupResponseBuilder = (message, { Config }, msgtype) => { + const { eventAsTags, allowedProperties } = Config; + const event = get(message, 'event'); + const groupId = getFieldValueFromMessage(message, 'groupId'); + // validation and adding tags for track and group call respectively + const tags = {}; + const payload = { properties: {} }; + if (msgtype === EventType.TRACK) { + if (!event) { + throw new InstrumentationError('Event is not present in the input payloads'); + } + /* Populating event as true in tags. + eg. tags: { + "event_name": true + } + */ + tags[event] = true; + payload.properties.purchases = getProductPurchasesDetails(message); + } + if (msgtype === EventType.GROUP) { + if (!groupId) { + throw new InstrumentationError('groupId is required for group events'); + } + tags.groupId = groupId; + } + + const externalUserId = getFieldValueFromMessage(message, 'userIdOnly'); + if (!externalUserId) { + const alias = getOneSignalAliases(message); + if (Object.keys(alias).length === 0) { + throw new InstrumentationError('userId or any other alias is required for track and group'); + } + payload.identity = alias; + } else { + payload.identity = { + external_id: externalUserId, + }; + } + + // Populating tags using allowed properties(from dashboard) + const properties = get(message, 'properties'); + if (properties && allowedProperties && Array.isArray(allowedProperties)) { + allowedProperties.forEach((item) => { + if (typeof properties[item.propertyName] === 'string') { + const tagName = + event && eventAsTags ? `${event}_${[item.propertyName]}` : item.propertyName; + tags[tagName] = properties[item.propertyName]; + } + }); + } + payload.properties.tags = tags; + return responseBuilder(removeUndefinedAndNullValues(payload), Config); +}; + +const processEvent = (message, destination) => { + if (!message.type) { + throw new InstrumentationError('Event type is required'); + } + if (!destination.Config.appId) { + throw new ConfigurationError('appId is a required field'); + } + const messageType = message.type.toLowerCase(); + let response; + switch (messageType) { + case EventType.IDENTIFY: + response = identifyResponseBuilder(message, destination); + break; + case EventType.TRACK: + response = trackOrGroupResponseBuilder(message, destination, EventType.TRACK); + break; + case EventType.GROUP: + response = trackOrGroupResponseBuilder(message, destination, EventType.GROUP); + break; + default: + throw new InstrumentationError(`Message type ${messageType} is not supported`); + } + return response; +}; + +const process = (message, destination) => processEvent(message, destination); + +module.exports = { process }; diff --git a/src/v0/destinations/one_signal/util.js b/src/v0/destinations/one_signal/util.js index 2de57de1b4..69cbd5440c 100644 --- a/src/v0/destinations/one_signal/util.js +++ b/src/v0/destinations/one_signal/util.js @@ -1,6 +1,13 @@ const { InstrumentationError } = require('@rudderstack/integrations-lib'); -const { getIntegrationsObj, getFieldValueFromMessage, getBrowserInfo } = require('../../util'); - +const { + getIntegrationsObj, + getFieldValueFromMessage, + getBrowserInfo, + constructPayload, + removeUndefinedAndNullValues, +} = require('../../util'); +const { ConfigCategory, mappingConfig, deviceTypesV2Enums } = require('./config'); +const { isDefinedAndNotNullAndNotEmpty } = require('../../util'); // For mapping device_type value const deviceTypeMapping = { android: 1, @@ -45,7 +52,7 @@ const populateTags = (message) => { const populateDeviceType = (message, payload) => { const integrationsObj = getIntegrationsObj(message, 'one_signal'); const devicePayload = payload; - if (integrationsObj && integrationsObj.deviceType && integrationsObj.identifier) { + if (integrationsObj?.deviceType && integrationsObj?.identifier) { devicePayload.device_type = parseInt(integrationsObj.deviceType, 10); devicePayload.identifier = integrationsObj.identifier; if (!validateDeviceType(devicePayload.device_type)) { @@ -72,4 +79,147 @@ const populateDeviceType = (message, payload) => { } }; -module.exports = { populateDeviceType, populateTags }; +/** + * This function is used to populate device type required for creating a subscription + * it checks from integrations object and fall back to message.channel and fif nothing is given it return a n empty object + * @param {*} message + * @param {*} payload + * returns Object + */ +const getDeviceDetails = (message) => { + const integrationsObj = getIntegrationsObj(message, 'one_signal'); + const devicePayload = {}; + if (integrationsObj?.deviceType && integrationsObj?.identifier) { + devicePayload.type = integrationsObj.deviceType; + devicePayload.token = integrationsObj.token || integrationsObj.identifier; + } + // Mapping device type when it is not present in the integrationsObject + if (!devicePayload.type) { + // if channel is mobile, checking for type from `context.device.type` + if (message.channel === 'mobile') { + devicePayload.type = message.context?.device?.type; + devicePayload.token = message.context?.device?.token + ? message.context.device.token + : message.context?.device?.id; + } + // Parsing the UA to get the browser info to map the device_type + if (message.channel === 'web' && message.context?.userAgent) { + const browser = getBrowserInfo(message.context.userAgent); + devicePayload.type = `${browser.name}Push`; // For chrome it would be like ChromePush + devicePayload.token = message.anonymousId; + } + } + if (!deviceTypesV2Enums.includes(devicePayload.type)) { + return {}; // No device related information available + } + return devicePayload; +}; +/** + * This function maps and returns the product purchases details built from input message.properties.products + * @param {*} message + * @returns + */ +const getProductPurchasesDetails = (message) => { + const { properties } = message; + const purchases = properties?.products; + if (purchases && Array.isArray(purchases)) { + return purchases.map((product) => ({ + sku: product.sku, + iso: product.iso, + count: product.quantity, + amount: product.amount, + })); + } + const purchaseObject = removeUndefinedAndNullValues({ + sku: properties?.sku, + iso: properties?.iso, + count: properties?.quantity, + amount: properties?.amount, + }); + return Object.keys(purchaseObject).length > 0 ? [purchaseObject] : []; +}; + +/** + * This function generates the subscriptions Payload for the given deviceType and token + * https://documentation.onesignal.com/reference/create-user#:~:text=string-,subscriptions,-array%20of%20objects + * @param {*} message + * @param {*} deviceType + * @param {*} token + * @returns + */ +const constructSubscription = (message, subscriptionType, token, subscriptionField) => { + const deviceModel = message.context?.device?.model; + const deviceOs = message.context?.os?.version; + let deviceSubscriptionPayload = { + type: subscriptionType, + token, + device_model: deviceModel, + device_os: deviceOs, + }; + // Following mapping is used to do paticular and specific property mapping for subscription + const traits = message.context?.traits || message.traits; + if (traits?.subscriptions?.[subscriptionField]) { + deviceSubscriptionPayload = { + ...deviceSubscriptionPayload, + ...constructPayload( + traits.subscriptions[subscriptionField], + mappingConfig[ConfigCategory.SUBSCRIPTION.name], + ), + }; + } + return deviceSubscriptionPayload; +}; + +/** + * This function constructs subscriptions list from message and returns subscriptions list + * @param {*} message + * @param {*} Config + * @returns + */ +const getSubscriptions = (message, Config) => { + const { emailDeviceType, smsDeviceType } = Config; + // Creating response for creation of new device or updation of an existing device + const subscriptions = []; + const deviceTypeSubscription = getDeviceDetails(message); + if (deviceTypeSubscription.token) { + subscriptions.push( + constructSubscription(message, deviceTypeSubscription.type, deviceTypeSubscription.token), + ); + } + + // Creating a device with email as an identifier + if (emailDeviceType) { + const token = getFieldValueFromMessage(message, 'email'); + if (isDefinedAndNotNullAndNotEmpty(token)) { + subscriptions.push(constructSubscription(message, 'Email', token, 'email')); + } + } + // Creating a device with phone as an identifier + if (smsDeviceType) { + const token = getFieldValueFromMessage(message, 'phone'); + if (isDefinedAndNotNullAndNotEmpty(token)) { + subscriptions.push(constructSubscription(message, 'SMS', token, 'phone')); + } + } + return subscriptions.length > 0 ? subscriptions : undefined; +}; + +/** + * This function fetched all the aliases to be passed to one signal from integrations object + * @param {*} message + * @returns object + */ +const getOneSignalAliases = (message) => { + const integrationsObj = getIntegrationsObj(message, 'one_signal'); + if (integrationsObj?.aliases) { + return integrationsObj.aliases; + } + return {}; +}; +module.exports = { + populateDeviceType, + populateTags, + getProductPurchasesDetails, + getSubscriptions, + getOneSignalAliases, +}; diff --git a/src/v0/destinations/one_signal/utils.test.js b/src/v0/destinations/one_signal/utils.test.js new file mode 100644 index 0000000000..afcf746ab6 --- /dev/null +++ b/src/v0/destinations/one_signal/utils.test.js @@ -0,0 +1,35 @@ +const { getOneSignalAliases } = require('./util'); + +describe('getOneSignalAliases', () => { + // returns aliases when integrationsObj contains aliases + it('should return aliases when integrationsObj contains aliases', () => { + const message = { + someKey: 'someValue', + integrations: { one_signal: { aliases: { alias1: 'value1' } } }, + }; + const result = getOneSignalAliases(message); + expect(result).toEqual({ alias1: 'value1' }); + }); + + // handles null or undefined message parameter gracefully + it('should handle null or undefined message parameter gracefully', () => { + let result = getOneSignalAliases(null); + expect(result).toEqual({}); + result = getOneSignalAliases(undefined); + expect(result).toEqual({}); + }); + + // returns an empty object when integrationsObj does not contain aliases + it('should return an empty object when integrationsObj does not contain aliases', () => { + const message = { someKey: 'someValue', integrations: { one_signal: {} } }; + const result = getOneSignalAliases(message); + expect(result).toEqual({}); + }); + + // handles message parameter with unexpected structure + it('should handle message parameter with unexpected structure', () => { + const message = null; + const result = getOneSignalAliases(message); + expect(result).toEqual({}); + }); +}); diff --git a/test/integrations/destinations/one_signal/processor/commonConfig.ts b/test/integrations/destinations/one_signal/processor/commonConfig.ts new file mode 100644 index 0000000000..cdef3dbfb4 --- /dev/null +++ b/test/integrations/destinations/one_signal/processor/commonConfig.ts @@ -0,0 +1,45 @@ +export const destination = { + Config: { + appId: 'random-818c-4a28-b98e-6cd8a994eb22', + emailDeviceType: true, + smsDeviceType: true, + eventAsTags: false, + allowedProperties: [ + { propertyName: 'brand' }, + { propertyName: 'firstName' }, + { propertyName: 'price' }, + ], + version: 'V2', + }, +}; + +export const endpoint = 'https://api.onesignal.com/apps/random-818c-4a28-b98e-6cd8a994eb22/users'; + +export const headers = { Accept: 'application/json', 'Content-Type': 'application/json' }; + +export const commonTraits = { + brand: 'John Players', + price: '15000', + firstName: 'Test', +}; +export const commonTags = { + brand: 'John Players', + price: '15000', + firstName: 'Test', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', +}; + +export const commonProperties = { + products: [ + { + sku: 3, + iso: 'iso', + quantity: 2, + amount: 100, + }, + ], + brand: 'John Players', + price: '15000', + firstName: 'Test', + customKey: 'customVal', +}; diff --git a/test/integrations/destinations/one_signal/processor/data.ts b/test/integrations/destinations/one_signal/processor/data.ts index 4171157aef..13aa6043f1 100644 --- a/test/integrations/destinations/one_signal/processor/data.ts +++ b/test/integrations/destinations/one_signal/processor/data.ts @@ -1,3 +1,4 @@ +import { data_v2 } from './data_v2'; export const data = [ { name: 'one_signal', @@ -1542,4 +1543,5 @@ export const data = [ }, }, }, + ...data_v2, ]; diff --git a/test/integrations/destinations/one_signal/processor/data_v2.ts b/test/integrations/destinations/one_signal/processor/data_v2.ts new file mode 100644 index 0000000000..325dca336d --- /dev/null +++ b/test/integrations/destinations/one_signal/processor/data_v2.ts @@ -0,0 +1,5 @@ +import { identifyTests } from './identify'; +import { trackTests } from './track'; +import { validations } from './validations'; +import { groupTests } from './group'; +export const data_v2 = [...identifyTests, ...trackTests, ...validations, ...groupTests]; diff --git a/test/integrations/destinations/one_signal/processor/group.ts b/test/integrations/destinations/one_signal/processor/group.ts new file mode 100644 index 0000000000..e16d1e4fe6 --- /dev/null +++ b/test/integrations/destinations/one_signal/processor/group.ts @@ -0,0 +1,121 @@ +import { destination, endpoint, headers } from './commonConfig'; +export const groupTests = [ + { + name: 'one_signal', + id: 'One Signal V2-test-group-success-1', + description: + 'Group call for adding a tag groupId with value as group id with no userId available', + module: 'destination', + successCriteria: 'Request gets passed with 200 Status Code with userId mapped to external_id', + feature: 'processor', + scenario: 'Framework+Buisness', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + integrations: { + one_signal: { + aliases: { custom_alias: 'custom_alias_identifier' }, + }, + }, + type: 'group', + channel: 'server', + groupId: 'group@27', + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params: {}, + body: { + FORM: {}, + JSON: { + properties: { + tags: { + groupId: 'group@27', + }, + }, + identity: { + custom_alias: 'custom_alias_identifier', + }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'one_signal', + id: 'One Signal V2-test-group-failure-1', + description: 'V2-> No Group Id Passes', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code and failure happened due no group id available', + feature: 'processor', + scenario: 'Framework', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'group', + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + error: 'groupId is required for group events', + statTags: { + destType: 'ONE_SIGNAL', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/one_signal/processor/identify.ts b/test/integrations/destinations/one_signal/processor/identify.ts new file mode 100644 index 0000000000..175c18a8fa --- /dev/null +++ b/test/integrations/destinations/one_signal/processor/identify.ts @@ -0,0 +1,341 @@ +import { commonTags, commonTraits, destination, endpoint, headers } from './commonConfig'; + +export const identifyTests = [ + { + name: 'one_signal', + id: 'One Signal V2-test-identify-success-1', + description: + 'V2-> Identify call for creating new user with userId only available and no subscriptions available', + module: 'destination', + successCriteria: 'Request gets passed with 200 Status Code with userId mapped to external_id', + feature: 'processor', + scenario: 'Framework+Buisness', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + userId: 'user@27', + channel: 'server', + context: { + app: { + version: '1.1.11', + }, + traits: commonTraits, + locale: 'en-US', + screen: { density: 2 }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', + }, + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params: {}, + body: { + FORM: {}, + JSON: { + properties: { + tags: commonTags, + laguage: 'en-US', + created_at: 1609693373, + last_active: 1609693373, + }, + identity: { + external_id: 'user@27', + }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'one_signal', + id: 'One Signal V2-test-identify-success-2', + description: + 'V2-> Identify call for creating new user with userId and one device subscription available', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code with userId mapped to external_id and one subscription for device where identifier is mapped from anonymousId', + feature: 'processor', + scenario: 'Framework+Buisness', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + userId: 'user@27', + channel: 'web', + context: { + app: { + name: 'RudderLabs JavaScript SDK', + build: '1.0.0', + version: '1.1.11', + namespace: 'com.rudderlabs.javascript', + }, + traits: commonTraits, + locale: 'en-US', + campaign: {}, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', + }, + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params: {}, + body: { + FORM: {}, + JSON: { + properties: { + tags: commonTags, + laguage: 'en-US', + created_at: 1609693373, + last_active: 1609693373, + }, + subscriptions: [ + { token: '97c46c81-3140-456d-b2a9-690d70aaca35', type: 'FirefoxPush' }, + ], + identity: { + external_id: 'user@27', + }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'one_signal', + id: 'One Signal V2-test-identify-success-3', + description: + 'V2-> Identify call for creating new user with userId and three device subscription available', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code with userId mapped to external_id and three subscription for device where one is mapped from anonymousId, one from email and one from phone', + feature: 'processor', + scenario: 'Framework+Buisness', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + one_signal: { + aliases: { custom_alias: 'custom_alias_identifier' }, + }, + }, + channel: 'web', + context: { + app: { + name: 'RudderLabs JavaScript SDK', + build: '1.0.0', + version: '1.1.11', + namespace: 'com.rudderlabs.javascript', + }, + device: { + model: 'dummy model', + }, + os: { version: '1.0.0' }, + traits: { + ...commonTraits, + email: 'example@abc.com', + phone: '12345678', + subscriptions: { + email: { + enabled: true, + notification_types: 'SMS', + session_time: 123456, + session_count: 22, + app_version: '1.0.0', + test_type: 'dev', + }, + }, + }, + locale: 'en-US', + campaign: {}, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', + }, + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params: {}, + body: { + FORM: {}, + JSON: { + properties: { + tags: { ...commonTags, email: 'example@abc.com', phone: '12345678' }, + laguage: 'en-US', + created_at: 1609693373, + last_active: 1609693373, + }, + subscriptions: [ + { + device_model: 'dummy model', + device_os: '1.0.0', + token: '97c46c81-3140-456d-b2a9-690d70aaca35', + type: 'FirefoxPush', + }, + { + app_version: '1.0.0', + device_model: 'dummy model', + device_os: '1.0.0', + enabled: true, + notification_types: 'SMS', + session_count: 22, + session_time: 123456, + test_type: 'dev', + token: 'example@abc.com', + type: 'Email', + }, + { + device_model: 'dummy model', + device_os: '1.0.0', + token: '12345678', + type: 'SMS', + }, + ], + identity: { custom_alias: 'custom_alias_identifier' }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'one_signal', + id: 'One Signal V2-test-identify-failure-1', + description: 'V2-> Identify call without any aliases', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code and failure happened due no aliases present', + feature: 'processor', + scenario: 'Framework', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + channel: 'server', + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + error: 'userId or any other alias is required for identify', + statTags: { + destType: 'ONE_SIGNAL', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/one_signal/processor/track.ts b/test/integrations/destinations/one_signal/processor/track.ts new file mode 100644 index 0000000000..41735fc7a2 --- /dev/null +++ b/test/integrations/destinations/one_signal/processor/track.ts @@ -0,0 +1,249 @@ +import { commonProperties, destination, endpoint, headers } from './commonConfig'; + +const commonTrackTags = { + brand: 'John Players', + price: '15000', + firstName: 'Test', +}; + +const purchases = [ + { + sku: 3, + iso: 'iso', + count: 2, + amount: 100, + }, +]; + +export const trackTests = [ + { + name: 'one_signal', + id: 'One Signal V2-test-track-success-1', + description: + 'V2-> Track call for updating user tags with userId available and products details available', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code with userId mapped to external_id and properties mapped to tags', + feature: 'processor', + scenario: 'Framework+Buisness', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + event: 'Order Completed', + userId: 'user@27', + channel: 'server', + properties: commonProperties, + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params: {}, + body: { + FORM: {}, + JSON: { + properties: { purchases, tags: { ...commonTrackTags, 'Order Completed': true } }, + + identity: { + external_id: 'user@27', + }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'one_signal', + id: 'One Signal V2-test-track-success-2', + description: 'V2-> Track call for products details available in properties directly', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code with userId mapped to external_id and purchases mapped from proeprties mapped to tags', + feature: 'processor', + scenario: 'Framework+Buisness', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + event: 'Order Completed', + userId: 'user@27', + channel: 'server', + properties: {}, + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params: {}, + body: { + FORM: {}, + JSON: { + properties: { purchases: [], tags: { 'Order Completed': true } }, + identity: { + external_id: 'user@27', + }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'one_signal', + id: 'One Signal V2-test-track-failure-1', + description: 'V2-> Track call without event name', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code and failure happened due instrumentation error', + feature: 'processor', + scenario: 'Framework+Buisness', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + userId: 'user@27', + channel: 'server', + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + error: 'Event is not present in the input payloads', + statTags: { + destType: 'ONE_SIGNAL', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + }, + { + name: 'one_signal', + id: 'One Signal V2-test-track-failure-2', + description: 'V2-> Track call without any aliases', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code and failure happened due no aliases present', + feature: 'processor', + scenario: 'Framework', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + event: 'dummy event', + channel: 'server', + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + error: 'userId or any other alias is required for track and group', + statTags: { + destType: 'ONE_SIGNAL', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/one_signal/processor/validations.ts b/test/integrations/destinations/one_signal/processor/validations.ts new file mode 100644 index 0000000000..7cfa158eee --- /dev/null +++ b/test/integrations/destinations/one_signal/processor/validations.ts @@ -0,0 +1,149 @@ +import { destination } from './commonConfig'; +export const validations = [ + { + name: 'one_signal', + id: 'One Signal V2-test-validation-failure-1', + description: 'V2-> No Message type passed', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code and failure happened due no message type present', + feature: 'processor', + scenario: 'Framework', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + userId: 'user@27', + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + error: 'Event type is required', + statTags: { + destType: 'ONE_SIGNAL', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + }, + { + name: 'one_signal', + id: 'One Signal V2-test-validation-failure-2', + description: 'V2-> invalid Message type passed', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code and failure happened due invalid message type present', + feature: 'processor', + scenario: 'Framework', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'invalid', + userId: 'user@27', + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + error: 'Message type invalid is not supported', + statTags: { + destType: 'ONE_SIGNAL', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + }, + { + name: 'one_signal', + id: 'One Signal V2-test-validation-failure-3', + description: 'V2-> No App Id Present in destination Config', + module: 'destination', + successCriteria: + 'Request gets passed with 200 Status Code and failure happened due no Configuration Error', + feature: 'processor', + scenario: 'Framework', + version: 'v0', + input: { + request: { + body: [ + { + destination: { Config: {} }, + message: { + type: 'invalid', + userId: 'user@27', + rudderId: '8f8fa6b5-8e24-489c-8e22-61f23f2e364f', + messageId: '2116ef8c-efc3-4ca4-851b-02ee60dad6ff', + anonymousId: '97c46c81-3140-456d-b2a9-690d70aaca35', + originalTimestamp: '2021-01-03T17:02:53.193Z', + }, + }, + ], + method: 'POST', + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + error: 'appId is a required field', + statTags: { + destType: 'ONE_SIGNAL', + errorCategory: 'dataValidation', + errorType: 'configuration', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + }, +]; From dc5b9fca7eee94af54cb604e72a177b9edf38e3d Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Tue, 25 Jun 2024 06:52:47 +0000 Subject: [PATCH 2/5] chore(release): 1.69.1 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42b22827da..dbe8ae98fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.69.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.69.0...v1.69.1) (2024-06-25) + + +### Bug Fixes + +* enhancement: introduce user model for one signal ([#3499](https://github.com/rudderlabs/rudder-transformer/issues/3499)) ([1c8e950](https://github.com/rudderlabs/rudder-transformer/commit/1c8e950f3d8789b33bba69a30c9eb21c40ce3d04)) + ## [1.69.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.68.2...v1.69.0) (2024-06-10) diff --git a/package-lock.json b/package-lock.json index 7fa0d6d3d4..c96e589aff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.69.0", + "version": "1.69.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.69.0", + "version": "1.69.1", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 2c7c6711e0..89513532e0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.69.0", + "version": "1.69.1", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 8129a06e3a12c9bca17354598849750498c72d2e Mon Sep 17 00:00:00 2001 From: Jayachand Date: Tue, 25 Jun 2024 15:32:23 +0530 Subject: [PATCH 3/5] fix: metadata tags capturing in v0 transformation (#3492) * fix: metadata tags capturing in v0 transformation --- src/util/customTransformer.js | 6 ++++-- src/v0/util/index.js | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/util/customTransformer.js b/src/util/customTransformer.js index 787ce04d63..4f4620fd2d 100644 --- a/src/util/customTransformer.js +++ b/src/util/customTransformer.js @@ -253,8 +253,10 @@ async function runUserTransform( const tags = { identifier: 'v0', errored: transformationError ? true : false, - ...(events.length && events[0].metadata ? getMetadata(events[0].metadata) : {}), - ...(events.length && events[0].metadata ? getTransformationMetadata(events[0].metadata) : {}), + ...(Object.keys(eventsMetadata).length ? getMetadata(Object.values(eventsMetadata)[0]) : {}), + ...(Object.keys(eventsMetadata).length + ? getTransformationMetadata(Object.values(eventsMetadata)[0]) + : {}), }; stats.counter('user_transform_function_input_events', events.length, tags); diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 366c58ce93..12b8d4dd7e 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -1447,13 +1447,13 @@ const getTrackingPlanMetadata = (metadata) => ({ workspaceId: metadata.workspaceId, }); -const getMetadata = (metadata) => ({ +const getMetadata = (metadata = {}) => ({ sourceType: metadata.sourceType, destinationType: metadata.destinationType, k8_namespace: metadata.namespace, }); -const getTransformationMetadata = (metadata) => ({ +const getTransformationMetadata = (metadata = {}) => ({ transformationId: metadata.transformationId, workspaceId: metadata.workspaceId, }); From 0379c4d8d5cf779f84e7d44a6d061ce988290e2e Mon Sep 17 00:00:00 2001 From: Jayachand Date: Tue, 25 Jun 2024 17:41:12 +0530 Subject: [PATCH 4/5] chore: remove version id, clean unused stats (#3494) * chore: remove versionId, unused or unimportant stats --- src/legacy/router.js | 18 +- src/util/customTransformer-faas.js | 2 +- src/util/customTransformer-v1.js | 1 - src/util/customTransformer.js | 22 +- src/util/customTransforrmationsStore-v1.js | 37 +- src/util/customTransforrmationsStore.js | 8 +- src/util/ivmFactory.js | 18 +- src/util/prometheus.js | 558 ++++++++------------- src/util/utils.js | 14 +- 9 files changed, 254 insertions(+), 424 deletions(-) diff --git a/src/legacy/router.js b/src/legacy/router.js index afc8c1a797..043e37b66d 100644 --- a/src/legacy/router.js +++ b/src/legacy/router.js @@ -539,9 +539,7 @@ if (startDestTransformer) { (event) => `${event.metadata.destinationId}_${event.metadata.sourceId}`, ); } - stats.counter('user_transform_function_group_size', Object.entries(groupedEvents).length, { - processSessions, - }); + stats.counter('user_transform_function_group_size', Object.entries(groupedEvents).length, {}); let ctxStatusCode = 200; const transformedEvents = []; @@ -646,16 +644,10 @@ if (startDestTransformer) { ctx.status = ctxStatusCode; ctx.set('apiVersion', API_VERSION); - stats.timing('user_transform_request_latency', startTime, { - processSessions, - }); - stats.timingSummary('user_transform_request_latency_summary', startTime, { - processSessions, - }); - stats.increment('user_transform_requests', { processSessions }); - stats.histogram('user_transform_output_events', transformedEvents.length, { - processSessions, - }); + stats.timing('user_transform_request_latency', startTime, {}); + stats.timingSummary('user_transform_request_latency_summary', startTime, {}); + stats.increment('user_transform_requests', {}); + stats.histogram('user_transform_output_events', transformedEvents.length, {}); }); } } diff --git a/src/util/customTransformer-faas.js b/src/util/customTransformer-faas.js index 9ac9804097..07dc205582 100644 --- a/src/util/customTransformer-faas.js +++ b/src/util/customTransformer-faas.js @@ -91,7 +91,7 @@ async function setOpenFaasUserTransform( trMetadata = {}, ) { const tags = { - transformerVersionId: userTransformation.versionId, + transformationId: userTransformation.id, identifier: 'openfaas', testMode, }; diff --git a/src/util/customTransformer-v1.js b/src/util/customTransformer-v1.js index b1865dee6b..12dab547e6 100644 --- a/src/util/customTransformer-v1.js +++ b/src/util/customTransformer-v1.js @@ -65,7 +65,6 @@ async function userTransformHandlerV1( const isolatevmFactory = await getFactory( userTransformation.code, libraryVersionIds, - userTransformation.versionId, userTransformation.id, userTransformation.workspaceId, credentialsMap, diff --git a/src/util/customTransformer.js b/src/util/customTransformer.js index 4f4620fd2d..5ca1fae47c 100644 --- a/src/util/customTransformer.js +++ b/src/util/customTransformer.js @@ -16,9 +16,11 @@ async function runUserTransform( code, secrets, eventsMetadata, - versionId, + transformationId, + workspaceId, testMode = false, ) { + const trTags = { identifier: 'v0', transformationId, workspaceId }; // TODO: Decide on the right value for memory limit const isolate = new ivm.Isolate({ memoryLimit: ISOLATE_VM_MEMORY }); const context = await isolate.createContext(); @@ -36,9 +38,9 @@ async function runUserTransform( new ivm.Reference(async (resolve, ...args) => { try { const fetchStartTime = new Date(); - const res = await fetchWithDnsWrapper(versionId, ...args); + const res = await fetchWithDnsWrapper(trTags, ...args); const data = await res.json(); - stats.timing('fetch_call_duration', fetchStartTime, { versionId }); + stats.timing('fetch_call_duration', fetchStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(data).copyInto()]); } catch (error) { resolve.applyIgnored(undefined, [new ivm.ExternalCopy('ERROR').copyInto()]); @@ -51,7 +53,7 @@ async function runUserTransform( new ivm.Reference(async (resolve, reject, ...args) => { try { const fetchStartTime = new Date(); - const res = await fetchWithDnsWrapper(versionId, ...args); + const res = await fetchWithDnsWrapper(trTags, ...args); const headersContent = {}; res.headers.forEach((value, header) => { headersContent[header] = value; @@ -67,7 +69,7 @@ async function runUserTransform( data.body = JSON.parse(data.body); } catch (e) {} - stats.timing('fetchV2_call_duration', fetchStartTime, { versionId }); + stats.timing('fetchV2_call_duration', fetchStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(data).copyInto()]); } catch (error) { const err = JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error))); @@ -93,7 +95,7 @@ async function runUserTransform( throw new Error(`request to fetch geolocation failed with status code: ${res.status}`); } const geoData = await res.json(); - stats.timing('geo_call_duration', geoStartTime, { versionId }); + stats.timing('geo_call_duration', geoStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(geoData).copyInto()]); } catch (error) { const err = JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error))); @@ -251,12 +253,9 @@ async function runUserTransform( isolate.dispose(); const tags = { - identifier: 'v0', errored: transformationError ? true : false, ...(Object.keys(eventsMetadata).length ? getMetadata(Object.values(eventsMetadata)[0]) : {}), - ...(Object.keys(eventsMetadata).length - ? getTransformationMetadata(Object.values(eventsMetadata)[0]) - : {}), + ...trTags, }; stats.counter('user_transform_function_input_events', events.length, tags); @@ -318,7 +317,8 @@ async function userTransformHandler( res.code, res.secrets || {}, eventsMetadata, - versionId, + res.id, + res.workspaceId, testMode, ); diff --git a/src/util/customTransforrmationsStore-v1.js b/src/util/customTransforrmationsStore-v1.js index 6e2d799f3a..d2d14f318e 100644 --- a/src/util/customTransforrmationsStore-v1.js +++ b/src/util/customTransforrmationsStore-v1.js @@ -19,25 +19,19 @@ const getRudderLibrariesUrl = `${CONFIG_BACKEND_URL}/rudderstackTransformationLi async function getTransformationCodeV1(versionId) { const transformation = transformationCache[versionId]; if (transformation) return transformation; - const tags = { - versionId, - version: 1, - }; try { const url = `${getTransformationURL}?versionId=${versionId}`; - const startTime = new Date(); const response = await fetchWithProxy(url); responseStatusHandler(response.status, 'Transformation', versionId, url); - stats.increment('get_transformation_code', { success: 'true', ...tags }); - stats.timing('get_transformation_code_time', startTime, tags); - stats.timingSummary('get_transformation_code_time_summary', startTime, tags); const myJson = await response.json(); transformationCache[versionId] = myJson; return myJson; } catch (error) { - logger.error(error); - stats.increment('get_transformation_code', { success: 'false', ...tags }); + logger.error( + `Error fetching transformation V1 code for versionId: ${versionId}`, + error.message, + ); throw error; } } @@ -45,25 +39,16 @@ async function getTransformationCodeV1(versionId) { async function getLibraryCodeV1(versionId) { const library = libraryCache[versionId]; if (library) return library; - const tags = { - libraryVersionId: versionId, - version: 1, - }; try { const url = `${getLibrariesUrl}?versionId=${versionId}`; - const startTime = new Date(); const response = await fetchWithProxy(url); responseStatusHandler(response.status, 'Transformation Library', versionId, url); - stats.increment('get_libraries_code', { success: 'true', ...tags }); - stats.timing('get_libraries_code_time', startTime, tags); - stats.timingSummary('get_libraries_code_time_summary', startTime, tags); const myJson = await response.json(); libraryCache[versionId] = myJson; return myJson; } catch (error) { - logger.error(error); - stats.increment('get_libraries_code', { success: 'false', ...tags }); + logger.error(`Error fetching library code for versionId: ${versionId}`, error.message); throw error; } } @@ -71,27 +56,17 @@ async function getLibraryCodeV1(versionId) { async function getRudderLibByImportName(importName) { const rudderLibrary = rudderLibraryCache[importName]; if (rudderLibrary) return rudderLibrary; - const tags = { - libraryVersionId: importName, - version: 1, - type: 'rudderlibrary', - }; try { const [name, version] = importName.split('/').slice(-2); const url = `${getRudderLibrariesUrl}/${name}?version=${version}`; - const startTime = new Date(); const response = await fetchWithProxy(url); responseStatusHandler(response.status, 'Rudder Library', importName, url); - stats.increment('get_libraries_code', { success: 'true', ...tags }); - stats.timing('get_libraries_code_time', startTime, tags); - stats.timingSummary('get_libraries_code_time_summary', startTime, tags); const myJson = await response.json(); rudderLibraryCache[importName] = myJson; return myJson; } catch (error) { - logger.error(error); - stats.increment('get_libraries_code', { success: 'false', ...tags }); + logger.error(`Error fetching rudder library code for importName: ${importName}`, error.message); throw error; } } diff --git a/src/util/customTransforrmationsStore.js b/src/util/customTransforrmationsStore.js index 2c5a7b446d..2043d18875 100644 --- a/src/util/customTransforrmationsStore.js +++ b/src/util/customTransforrmationsStore.js @@ -2,7 +2,6 @@ const NodeCache = require('node-cache'); const { fetchWithProxy } = require('./fetch'); const logger = require('../logger'); const { responseStatusHandler } = require('./utils'); -const stats = require('./stats'); const myCache = new NodeCache({ stdTTL: 60 * 60 * 24 * 1 }); @@ -18,19 +17,14 @@ async function getTransformationCode(versionId) { if (transformation) return transformation; try { const url = `${getTransformationURL}?versionId=${versionId}`; - const startTime = new Date(); const response = await fetchWithProxy(url); responseStatusHandler(response.status, 'Transformation', versionId, url); - stats.increment('get_transformation_code', { versionId, success: 'true' }); - stats.timing('get_transformation_code_time', startTime, { versionId }); - stats.timingSummary('get_transformation_code_time_summary', startTime, { versionId }); const myJson = await response.json(); myCache.set(versionId, myJson); return myJson; } catch (error) { - logger.error(error); - stats.increment('get_transformation_code', { versionId, success: 'false' }); + logger.error(`Error fetching transformation code for versionId: ${versionId}`, error.message); throw error; } } diff --git a/src/util/ivmFactory.js b/src/util/ivmFactory.js index 44beff01dc..625591964c 100644 --- a/src/util/ivmFactory.js +++ b/src/util/ivmFactory.js @@ -33,13 +33,13 @@ async function loadModule(isolateInternal, contextInternal, moduleName, moduleCo async function createIvm( code, libraryVersionIds, - versionId, transformationId, workspaceId, credentials, secrets, testMode, ) { + const trTags = { identifier: 'V1', transformationId, workspaceId }; const createIvmStartTime = new Date(); const logs = []; const libraries = await Promise.all( @@ -187,9 +187,9 @@ async function createIvm( new ivm.Reference(async (resolve, ...args) => { try { const fetchStartTime = new Date(); - const res = await fetchWithDnsWrapper(versionId, ...args); + const res = await fetchWithDnsWrapper(trTags, ...args); const data = await res.json(); - stats.timing('fetch_call_duration', fetchStartTime, { versionId }); + stats.timing('fetch_call_duration', fetchStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(data).copyInto()]); } catch (error) { resolve.applyIgnored(undefined, [new ivm.ExternalCopy('ERROR').copyInto()]); @@ -202,7 +202,7 @@ async function createIvm( new ivm.Reference(async (resolve, reject, ...args) => { try { const fetchStartTime = new Date(); - const res = await fetchWithDnsWrapper(versionId, ...args); + const res = await fetchWithDnsWrapper(trTags, ...args); const headersContent = {}; res.headers.forEach((value, header) => { headersContent[header] = value; @@ -218,7 +218,7 @@ async function createIvm( data.body = JSON.parse(data.body); } catch (e) {} - stats.timing('fetchV2_call_duration', fetchStartTime, { versionId }); + stats.timing('fetchV2_call_duration', fetchStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(data).copyInto()]); } catch (error) { const err = JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error))); @@ -243,7 +243,7 @@ async function createIvm( throw new Error(`request to fetch geolocation failed with status code: ${res.status}`); } const geoData = await res.json(); - stats.timing('geo_call_duration', geoStartTime, { versionId }); + stats.timing('geo_call_duration', geoStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(geoData).copyInto()]); } catch (error) { const err = JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error))); @@ -257,7 +257,7 @@ async function createIvm( logger.error( `Error fetching credentials map for transformationID: ${transformationId} and workspaceId: ${workspaceId}`, ); - stats.increment('credential_error_total', { transformationId, workspaceId }); + stats.increment('credential_error_total', trTags); return undefined; } if (key === null || key === undefined) { @@ -416,7 +416,7 @@ async function createIvm( reference: true, }); const fName = availableFuncNames[0]; - stats.timing('createivm_duration', createIvmStartTime); + stats.timing('createivm_duration', createIvmStartTime, trTags); // TODO : check if we can resolve this // eslint-disable-next-line no-async-promise-executor @@ -446,7 +446,6 @@ async function getFactory( libraryVersionIds, transformationId, workspaceId, - versionId, credentials, secrets, testMode, @@ -456,7 +455,6 @@ async function getFactory( return createIvm( code, libraryVersionIds, - versionId, transformationId, workspaceId, credentials, diff --git a/src/util/prometheus.js b/src/util/prometheus.js index ffcfda3784..860c266565 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -239,79 +239,12 @@ class Prometheus { 'implementation', ], }, - { - name: 'tp_violation_type', - help: 'tp_violation_type', - type: 'counter', - labelNames: ['violationType', 'sourceType', 'destinationType', 'k8_namespace'], - }, - { - name: 'tp_propagated_events', - help: 'tp_propagated_events', - type: 'counter', - labelNames: ['sourceType', 'destinationType', 'k8_namespace'], - }, - { - name: 'tp_errors', - help: 'tp_errors', - type: 'counter', - labelNames: [ - 'sourceType', - 'destinationType', - 'k8_namespace', - 'workspaceId', - 'trackingPlanId', - ], - }, - { - name: 'tp_events_count', - help: 'tp_events_count', - type: 'counter', - labelNames: ['sourceType', 'destinationType', 'k8_namespace'], - }, - { - name: 'user_transform_function_group_size', - help: 'user_transform_function_group_size', - type: 'counter', - labelNames: ['processSessions'], - }, - { - name: 'user_transform_errors', - help: 'user_transform_errors', - type: 'counter', - labelNames: [ - 'workspaceId', - 'transformationId', - 'status', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, - { - name: 'c2', - help: 'h2', - type: 'counter', - labelNames: [ - 'transformationVersionId', - 'processSessions', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, { name: 'dest_transform_requests', help: 'dest_transform_requests', type: 'counter', labelNames: ['destination', 'version', 'sourceType', 'destinationType', 'k8_namespace'], }, - { - name: 'user_transform_requests', - help: 'user_transform_requests', - type: 'counter', - labelNames: ['processSessions'], - }, { name: 'source_transform_requests', help: 'source_transform_requests', @@ -342,56 +275,6 @@ class Prometheus { type: 'counter', labelNames: ['success'], }, - { - name: 'create_zip_error', - help: 'create_zip_error', - type: 'counter', - labelNames: ['fileName'], - }, - { - name: 'delete_zip_error', - help: 'delete_zip_error', - type: 'counter', - labelNames: ['functionName'], - }, - { - name: 'hv_metrics', - help: 'hv_metrics', - type: 'counter', - labelNames: [ - 'destination', - 'version', - 'sourceType', - 'destinationType', - 'k8_namespace', - 'dropped', - 'violationType', - ], - }, - { - name: 'events_into_vm', - help: 'events_into_vm', - type: 'counter', - labelNames: [ - 'transformerVersionId', - 'version', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, - { - name: 'missing_handle', - help: 'missing_handle', - type: 'counter', - labelNames: [ - 'transformerVersionId', - 'language', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, { name: 'proxy_test_error', help: 'proxy_test_error', @@ -495,18 +378,6 @@ class Prometheus { type: 'counter', labelNames: ['destinationId'], }, - { - name: 'get_eventSchema_error', - help: 'get_eventSchema_error', - type: 'counter', - labelNames: [], - }, - { - name: 'get_tracking_plan_error', - help: 'get_tracking_plan_error', - type: 'counter', - labelNames: [], - }, { name: 'redis_error', help: 'redis_error', @@ -537,18 +408,6 @@ class Prometheus { type: 'counter', labelNames: ['writeKey', 'source'], }, - { - name: 'get_transformation_code', - help: 'get_transformation_code', - type: 'counter', - labelNames: ['versionId', 'version', 'success'], - }, - { - name: 'get_libraries_code', - help: 'get_libraries_code', - type: 'counter', - labelNames: ['libraryVersionId', 'version', 'type', 'success'], - }, { name: 'invalid_shopify_event', help: 'invalid_shopify_event', @@ -574,12 +433,6 @@ class Prometheus { 'sourceId', ], }, - { - name: 'credential_error_total', - help: 'Error in fetching credentials count', - type: 'counter', - labelNames: ['transformationId', 'workspaceId'], - }, // Gauges { @@ -667,118 +520,18 @@ class Prometheus { type: 'histogram', labelNames: ['method', 'route', 'code', 'destType'], }, - { - name: 'tp_batch_size', - help: 'Size of batch of events for tracking plan validation', - type: 'histogram', - buckets: [ - 1024, 102400, 524288, 1048576, 10485760, 20971520, 52428800, 104857600, 209715200, - 524288000, - ], - labelNames: [ - 'sourceType', - 'destinationType', - 'k8_namespace', - 'workspaceId', - 'trackingPlanId', - ], - }, - { - name: 'tp_event_validation_latency', - help: 'Latency of validating tracking plan at event level', - type: 'histogram', - labelNames: [ - 'sourceType', - 'destinationType', - 'k8_namespace', - 'workspaceId', - 'trackingPlanId', - 'status', - 'exception', - ], - }, - { - name: 'tp_batch_validation_latency', - help: 'Latency of validating tracking plan at batch level', - type: 'histogram', - labelNames: [ - 'sourceType', - 'destinationType', - 'k8_namespace', - 'workspaceId', - 'trackingPlanId', - ], - }, { name: 'cdk_events_latency', help: 'cdk_events_latency', type: 'histogram', labelNames: ['destination', 'sourceType', 'destinationType', 'k8_namespace'], }, - { - name: 'tp_event_latency', - help: 'tp_event_latency', - type: 'histogram', - labelNames: ['sourceType', 'destinationType', 'k8_namespace'], - }, { name: 'regulation_worker_requests_dest_latency', help: 'regulation_worker_requests_dest_latency', type: 'histogram', labelNames: ['feature', 'implementation', 'destType'], }, - { - name: 'user_transform_request_latency', - help: 'user_transform_request_latency', - type: 'histogram', - labelNames: [ - 'workspaceId', - 'transformationId', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, - { - name: 'user_transform_request_latency_summary', - help: 'user_transform_request_latency_summary', - type: 'summary', - labelNames: [ - 'workspaceId', - 'transformationId', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, - { - name: 'user_transform_batch_size', - help: 'user_transform_batch_size', - type: 'histogram', - labelNames: [ - 'workspaceId', - 'transformationId', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - buckets: [ - 1024, 102400, 524288, 1048576, 10485760, 20971520, 52428800, 104857600, 209715200, - 524288000, - ], // 1KB, 100KB, 0.5MB, 1MB, 10MB, 20MB, 50MB, 100MB, 200MB, 500MB - }, - { - name: 'user_transform_batch_size_summary', - help: 'user_transform_batch_size_summary', - type: 'summary', - labelNames: [ - 'workspaceId', - 'transformationId', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, { name: 'source_transform_request_latency', help: 'source_transform_request_latency', @@ -797,88 +550,6 @@ class Prometheus { type: 'histogram', labelNames: ['destination', 'version'], }, - { - name: 'creation_time', - help: 'creation_time', - type: 'histogram', - labelNames: ['transformerVersionId', 'language', 'identifier', 'publish', 'testMode'], - }, - { name: 'get_tracking_plan', help: 'get_tracking_plan', type: 'histogram', labelNames: [] }, - { name: 'createivm_duration', help: 'createivm_duration', type: 'histogram', labelNames: [] }, - { - name: 'fetchV2_call_duration', - help: 'fetchV2_call_duration', - type: 'histogram', - labelNames: ['versionId'], - }, - { - name: 'fetch_call_duration', - help: 'fetch_call_duration', - type: 'histogram', - labelNames: ['versionId'], - }, - { - name: 'fetch_dns_resolve_time', - help: 'fetch_dns_resolve_time', - type: 'histogram', - labelNames: ['transformerVersionId', 'error'], - }, - { - name: 'geo_call_duration', - help: 'geo_call_duration', - type: 'histogram', - labelNames: ['versionId'], - }, - { - name: 'get_transformation_code_time', - help: 'get_transformation_code_time', - type: 'histogram', - labelNames: ['versionId', 'version'], - }, - { - name: 'get_transformation_code_time_summary', - help: 'get_transformation_code_time_summary', - type: 'summary', - labelNames: ['versionId', 'version'], - }, - { - name: 'get_libraries_code_time', - help: 'get_libraries_code_time', - type: 'histogram', - labelNames: ['libraryVersionId', 'versionId', 'type', 'version'], - }, - { - name: 'get_libraries_code_time_summary', - help: 'get_libraries_code_time_summary', - type: 'summary', - labelNames: ['libraryVersionId', 'versionId', 'type', 'version'], - }, - { - name: 'isolate_cpu_time', - help: 'isolate_cpu_time', - type: 'histogram', - labelNames: [ - 'transformerVersionId', - 'identifier', - 'version', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, - { - name: 'isolate_wall_time', - help: 'isolate_wall_time', - type: 'histogram', - labelNames: [ - 'transformerVersionId', - 'identifier', - 'version', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, { name: 'marketo_bulk_upload_process_time', help: 'marketo_bulk_upload_process_time', @@ -1004,20 +675,6 @@ class Prometheus { labelNames: ['destination', 'version', 'sourceType', 'destinationType', 'k8_namespace'], buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], }, - { - name: 'user_transform_input_events', - help: 'Number of input events to user transform', - type: 'histogram', - labelNames: ['processSessions'], - buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], - }, - { - name: 'user_transform_output_events', - help: 'user_transform_output_events', - type: 'histogram', - labelNames: ['processSessions'], - buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], - }, { name: 'marketo_bulk_upload_create_header_time', help: 'marketo_bulk_upload_create_header_time', @@ -1054,6 +711,117 @@ class Prometheus { type: 'histogram', labelNames: [], }, + // tracking plan metrics: + // counter + { + name: 'hv_metrics', + help: 'hv_metrics', + type: 'counter', + labelNames: [ + 'destination', + 'version', + 'sourceType', + 'destinationType', + 'k8_namespace', + 'dropped', + 'violationType', + ], + }, + { + name: 'get_eventSchema_error', + help: 'get_eventSchema_error', + type: 'counter', + labelNames: [], + }, + { + name: 'get_tracking_plan_error', + help: 'get_tracking_plan_error', + type: 'counter', + labelNames: [], + }, + // histogram + { + name: 'tp_batch_size', + help: 'Size of batch of events for tracking plan validation', + type: 'histogram', + buckets: [ + 1024, 102400, 524288, 1048576, 10485760, 20971520, 52428800, 104857600, 209715200, + 524288000, + ], + labelNames: [ + 'sourceType', + 'destinationType', + 'k8_namespace', + 'workspaceId', + 'trackingPlanId', + ], + }, + { + name: 'tp_event_validation_latency', + help: 'Latency of validating tracking plan at event level', + type: 'histogram', + labelNames: [ + 'sourceType', + 'destinationType', + 'k8_namespace', + 'workspaceId', + 'trackingPlanId', + 'status', + 'exception', + ], + }, + { + name: 'tp_batch_validation_latency', + help: 'Latency of validating tracking plan at batch level', + type: 'histogram', + labelNames: [ + 'sourceType', + 'destinationType', + 'k8_namespace', + 'workspaceId', + 'trackingPlanId', + ], + }, + { + name: 'tp_event_latency', + help: 'tp_event_latency', + type: 'histogram', + labelNames: ['sourceType', 'destinationType', 'k8_namespace'], + }, + { name: 'get_tracking_plan', help: 'get_tracking_plan', type: 'histogram', labelNames: [] }, + // User transform metrics + // counter + { + name: 'user_transform_function_group_size', + help: 'user_transform_function_group_size', + type: 'counter', + labelNames: [], + }, + { + name: 'user_transform_errors', + help: 'user_transform_errors', + type: 'counter', + labelNames: [ + 'workspaceId', + 'transformationId', + 'status', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + }, + { + name: 'user_transform_requests', + help: 'user_transform_requests', + type: 'counter', + labelNames: [], + }, + { + name: 'credential_error_total', + help: 'Error in fetching credentials count', + type: 'counter', + labelNames: ['identifier', 'transformationId', 'workspaceId'], + }, { name: 'user_transform_function_input_events', help: 'user_transform_function_input_events', @@ -1070,6 +838,85 @@ class Prometheus { 'workspaceId', ], }, + // histogram + { + name: 'user_transform_request_latency', + help: 'user_transform_request_latency', + type: 'histogram', + labelNames: [ + 'workspaceId', + 'transformationId', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + }, + { + name: 'user_transform_batch_size', + help: 'user_transform_batch_size', + type: 'histogram', + labelNames: [ + 'workspaceId', + 'transformationId', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + buckets: [ + 1024, 102400, 524288, 1048576, 10485760, 20971520, 52428800, 104857600, 209715200, + 524288000, + ], // 1KB, 100KB, 0.5MB, 1MB, 10MB, 20MB, 50MB, 100MB, 200MB, 500MB + }, + { + name: 'creation_time', + help: 'creation_time', + type: 'histogram', + labelNames: ['transformationId', 'identifier', 'testMode'], + }, + { + name: 'createivm_duration', + help: 'createivm_duration', + type: 'histogram', + labelNames: ['identifier', 'transformationId', 'workspaceId'], + }, + { + name: 'fetchV2_call_duration', + help: 'fetchV2_call_duration', + type: 'histogram', + labelNames: ['identifier', 'transformationId', 'workspaceId'], + }, + { + name: 'fetch_call_duration', + help: 'fetch_call_duration', + type: 'histogram', + labelNames: ['identifier', 'transformationId', 'workspaceId'], + }, + { + name: 'fetch_dns_resolve_time', + help: 'fetch_dns_resolve_time', + type: 'histogram', + labelNames: ['identifier', 'transformationId', 'workspaceId', 'error'], + }, + { + name: 'geo_call_duration', + help: 'geo_call_duration', + type: 'histogram', + labelNames: ['identifier', 'transformationId', 'workspaceId'], + }, + { + name: 'user_transform_input_events', + help: 'Number of input events to user transform', + type: 'histogram', + labelNames: [], + buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], + }, + { + name: 'user_transform_output_events', + help: 'user_transform_output_events', + type: 'histogram', + labelNames: [], + buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], + }, { name: 'user_transform_function_latency', help: 'user_transform_function_latency', @@ -1086,6 +933,31 @@ class Prometheus { 'workspaceId', ], }, + // summary + { + name: 'user_transform_request_latency_summary', + help: 'user_transform_request_latency_summary', + type: 'summary', + labelNames: [ + 'workspaceId', + 'transformationId', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + }, + { + name: 'user_transform_batch_size_summary', + help: 'user_transform_batch_size_summary', + type: 'summary', + labelNames: [ + 'workspaceId', + 'transformationId', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + }, { name: 'user_transform_function_latency_summary', help: 'user_transform_function_latency_summary', diff --git a/src/util/utils.js b/src/util/utils.js index 1ac70b9541..eb5a011444 100644 --- a/src/util/utils.js +++ b/src/util/utils.js @@ -16,7 +16,7 @@ const LOCAL_HOST_NAMES_LIST = ['localhost', '127.0.0.1', '[::]', '[::1]']; const LOCALHOST_OCTET = '127.'; const RECORD_TYPE_A = 4; // ipv4 -const staticLookup = (transformerVersionId) => async (hostname, _, cb) => { +const staticLookup = (transformationTags) => async (hostname, _, cb) => { let ips; const resolveStartTime = new Date(); try { @@ -24,13 +24,13 @@ const staticLookup = (transformerVersionId) => async (hostname, _, cb) => { } catch (error) { logger.error(`DNS Error Code: ${error.code} | Message : ${error.message}`); stats.timing('fetch_dns_resolve_time', resolveStartTime, { - transformerVersionId, + ...transformationTags, error: 'true', }); cb(null, `unable to resolve IP address for ${hostname}`, RECORD_TYPE_A); return; } - stats.timing('fetch_dns_resolve_time', resolveStartTime, { transformerVersionId }); + stats.timing('fetch_dns_resolve_time', resolveStartTime, transformationTags); if (ips.length === 0) { cb(null, `resolved empty list of IP address for ${hostname}`, RECORD_TYPE_A); @@ -48,9 +48,9 @@ const staticLookup = (transformerVersionId) => async (hostname, _, cb) => { cb(null, ips[0], RECORD_TYPE_A); }; -const httpAgentWithDnsLookup = (scheme, transformerVersionId) => { +const httpAgentWithDnsLookup = (scheme, transformationTags) => { const httpModule = scheme === 'http' ? http : https; - return new httpModule.Agent({ lookup: staticLookup(transformerVersionId) }); + return new httpModule.Agent({ lookup: staticLookup(transformationTags) }); }; const blockLocalhostRequests = (url) => { @@ -74,7 +74,7 @@ const blockInvalidProtocolRequests = (url) => { } }; -const fetchWithDnsWrapper = async (transformerVersionId, ...args) => { +const fetchWithDnsWrapper = async (transformationTags, ...args) => { if (process.env.DNS_RESOLVE_FETCH_HOST !== 'true') { return await fetch(...args); } @@ -88,7 +88,7 @@ const fetchWithDnsWrapper = async (transformerVersionId, ...args) => { const fetchOptions = args[1] || {}; const schemeName = fetchURL.startsWith('https') ? 'https' : 'http'; // assign resolved agent to fetch - fetchOptions.agent = httpAgentWithDnsLookup(schemeName, transformerVersionId); + fetchOptions.agent = httpAgentWithDnsLookup(schemeName, transformationTags); return await fetch(fetchURL, fetchOptions); }; From dc83798f1e02b1116ca4704a37c96b18db253e99 Mon Sep 17 00:00:00 2001 From: Sandeep Digumarty Date: Wed, 26 Jun 2024 10:14:14 +0530 Subject: [PATCH 5/5] chore: resolve bugsnag issue in braze (#3474) * chore: resolve bugsnag issue in braze * chore: resolve bugsnag issue in braze * chore: added tests * chore: added tests * fix: instead of throwing error now ignoring the undefined value in the products array --- src/v0/destinations/braze/util.js | 4 +- .../destinations/braze/processor/data.ts | 214 ++++++++++++++++++ 2 files changed, 217 insertions(+), 1 deletion(-) diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js index b3b29cdf96..45063d0ba2 100644 --- a/src/v0/destinations/braze/util.js +++ b/src/v0/destinations/braze/util.js @@ -570,7 +570,8 @@ function getPurchaseObjs(message, config) { 'Invalid Order Completed event: Properties object is missing in the message', ); } - const { products, currency: currencyCode } = properties; + const { currency: currencyCode } = properties; + let { products } = properties; if (!products) { throw new InstrumentationError( 'Invalid Order Completed event: Products array is missing in the message', @@ -581,6 +582,7 @@ function getPurchaseObjs(message, config) { throw new InstrumentationError('Invalid Order Completed event: Products is not an array'); } + products = products.filter((product) => isDefinedAndNotNull(product)); if (products.length === 0) { throw new InstrumentationError('Invalid Order Completed event: Products array is empty'); } diff --git a/test/integrations/destinations/braze/processor/data.ts b/test/integrations/destinations/braze/processor/data.ts index dc5a84470f..644bebace1 100644 --- a/test/integrations/destinations/braze/processor/data.ts +++ b/test/integrations/destinations/braze/processor/data.ts @@ -4111,4 +4111,218 @@ export const data = [ }, }, }, + { + name: 'braze', + description: 'Test 30', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'track', + event: 'Order Completed', + sentAt: '2020-09-14T12:09:37.491Z', + userId: 'Randomuser2222', + channel: 'web', + context: { + os: { + name: '', + version: '', + }, + app: { + name: 'RudderLabs JavaScript SDK', + build: '1.0.0', + version: '1.1.3', + namespace: 'com.rudderlabs.javascript', + }, + page: { + url: 'file:///Users/manashi/Desktop/rudder-all-sdk-application-testing/Fullstory%20test%20By%20JS%20SDK/braze.html', + path: '/Users/manashi/Desktop/rudder-all-sdk-application-testing/Fullstory%20test%20By%20JS%20SDK/braze.html', + title: 'Fullstory Test', + search: '', + referrer: '', + }, + locale: 'en-GB', + screen: { + density: 2, + }, + traits: { + email: 'manashi@gmaiol.com', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.3', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36', + }, + messageId: '24ecc509-ce3e-473c-8483-ba1ea2c195cb', + properties: { + products: [ + null, + { + sku: '45790-32', + url: 'https://www.example.com/product/path', + key1: { + key11: 'value1', + key22: 'value2', + }, + name: 'Monopoly: 3rd Edition', + price: 19, + category: 'Games', + quantity: 1, + image_url: 'https:///www.example.com/product/path.jpg', + currency78: 'USD', + product_id: '507f1f77bcf86cd799439011', + }, + ], + }, + anonymousId: 'c6ff1462-b692-43d6-8f6a-659efedc99ea', + integrations: { + All: true, + }, + originalTimestamp: '2020-09-14T12:09:37.491Z', + }, + destination: { + Config: { + restApiKey: 'dummyApiKey', + prefixProperties: true, + useNativeSDK: false, + }, + DestinationDefinition: { + DisplayName: 'Braze', + ID: '1WhbSZ6uA3H5ChVifHpfL2H6sie', + Name: 'BRAZE', + }, + Enabled: true, + ID: '1WhcOCGgj9asZu850HvugU2C3Aq', + Name: 'Braze', + Transformations: [], + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 400, + error: + 'Invalid Order Completed event: Message properties and product at index: 0 is missing currency', + statTags: { + errorCategory: 'dataValidation', + errorType: 'instrumentation', + destType: 'BRAZE', + module: 'destination', + implementation: 'native', + feature: 'processor', + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'Test 31', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'track', + event: 'Order Completed', + sentAt: '2020-09-14T12:09:37.491Z', + userId: 'Randomuser2222', + channel: 'web', + context: { + os: { + name: '', + version: '', + }, + app: { + name: 'RudderLabs JavaScript SDK', + build: '1.0.0', + version: '1.1.3', + namespace: 'com.rudderlabs.javascript', + }, + page: { + url: 'file:///Users/manashi/Desktop/rudder-all-sdk-application-testing/Fullstory%20test%20By%20JS%20SDK/braze.html', + path: '/Users/manashi/Desktop/rudder-all-sdk-application-testing/Fullstory%20test%20By%20JS%20SDK/braze.html', + title: 'Fullstory Test', + search: '', + referrer: '', + }, + locale: 'en-GB', + screen: { + density: 2, + }, + traits: { + email: 'manashi@gmaiol.com', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.3', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36', + }, + messageId: '24ecc509-ce3e-473c-8483-ba1ea2c195cb', + properties: { + products: [undefined], + }, + anonymousId: 'c6ff1462-b692-43d6-8f6a-659efedc99ea', + integrations: { + All: true, + }, + originalTimestamp: '2020-09-14T12:09:37.491Z', + }, + destination: { + Config: { + restApiKey: 'dummyApiKey', + prefixProperties: true, + useNativeSDK: false, + }, + DestinationDefinition: { + DisplayName: 'Braze', + ID: '1WhbSZ6uA3H5ChVifHpfL2H6sie', + Name: 'BRAZE', + }, + Enabled: true, + ID: '1WhcOCGgj9asZu850HvugU2C3Aq', + Name: 'Braze', + Transformations: [], + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 400, + error: 'Invalid Order Completed event: Products array is empty', + statTags: { + errorCategory: 'dataValidation', + errorType: 'instrumentation', + destType: 'BRAZE', + module: 'destination', + implementation: 'native', + feature: 'processor', + }, + }, + ], + }, + }, + }, ];