From 76e7bd4d5d3dd0da2e9d82aa8168b4d2b04e3bf1 Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Fri, 19 Jul 2024 13:41:50 +0530 Subject: [PATCH 01/28] chore: add error handling for ga4 v2 custom mappings (#3572) --- src/v0/destinations/ga4_v2/customMappingsHandler.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/ga4_v2/customMappingsHandler.js b/src/v0/destinations/ga4_v2/customMappingsHandler.js index 0f20788076..9e82788150 100644 --- a/src/v0/destinations/ga4_v2/customMappingsHandler.js +++ b/src/v0/destinations/ga4_v2/customMappingsHandler.js @@ -14,7 +14,7 @@ const { prepareUserProperties, sanitizeUserProperties, } = require('../ga4/utils'); -const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); const { removeUndefinedAndNullRecurse, constructPayload, @@ -117,7 +117,12 @@ const handleCustomMappings = (message, Config) => { const eventPropertiesMappings = mapping.eventProperties || []; - const ga4MappedPayload = applyCustomMappings(message, eventPropertiesMappings); + let ga4MappedPayload = {}; + try { + ga4MappedPayload = applyCustomMappings(message, eventPropertiesMappings); + } catch (e) { + throw new ConfigurationError(`[GA4]:: Error in custom mappings: ${e.message}`); + } removeUndefinedAndNullRecurse(ga4MappedPayload); From f840d54dcbdc011eeb716dce74f2ecb36e99d0e9 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:15:02 +0530 Subject: [PATCH 02/28] fix: job ordering for hs (#3319) * fix: job ordering for hs, initial commit * chore: test * fix: code changes to maintain order * chore: eslint fix * chore: eslint fix+1 * chore: add metic for batch size * chore: address comments Co-authored-by: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> * chore: update src/v0/destinations/hs/HSTransform-v2.js * chore: remove optional chaining --------- Co-authored-by: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> Co-authored-by: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Co-authored-by: Anant Jain Co-authored-by: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> --- sample.env | 2 +- src/util/prometheus.js | 6 + src/v0/destinations/hs/HSTransform-v2.js | 7 +- src/v0/destinations/hs/transform.js | 47 +- .../destinations/hs/router/config.ts | 57 + .../destinations/hs/router/data.ts | 1043 +++++++++++++---- 6 files changed, 944 insertions(+), 218 deletions(-) create mode 100644 test/integrations/destinations/hs/router/config.ts diff --git a/sample.env b/sample.env index 71b11fbfca..41caa84a10 100644 --- a/sample.env +++ b/sample.env @@ -9,4 +9,4 @@ REDIS_PORT = 6379 REDIS_PASSWORD = 123 REDIS_USERNAME = abc USE_REDIS_DB = true -REDIS_EXPIRY_TIME_IN_SEC = 3600 \ No newline at end of file +REDIS_EXPIRY_TIME_IN_SEC = 3600 diff --git a/src/util/prometheus.js b/src/util/prometheus.js index 0b1f3b34bf..49f1fdcd8b 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -502,6 +502,12 @@ class Prometheus { type: 'counter', labelNames: ['destination_id'], }, + { + name: 'hs_batch_size', + help: 'hs_batch_size', + type: 'gauge', + labelNames: ['destination_id'], + }, { name: 'mixpanel_batch_engage_pack_size', help: 'mixpanel_batch_engage_pack_size', diff --git a/src/v0/destinations/hs/HSTransform-v2.js b/src/v0/destinations/hs/HSTransform-v2.js index d2b26f1ab8..71f080205a 100644 --- a/src/v0/destinations/hs/HSTransform-v2.js +++ b/src/v0/destinations/hs/HSTransform-v2.js @@ -20,6 +20,7 @@ const { getDestinationExternalIDInfoForRetl, getDestinationExternalIDObjectForRetl, } = require('../../util'); +const stats = require('../../../util/stats'); const { IDENTIFY_CRM_UPDATE_CONTACT, IDENTIFY_CRM_CREATE_NEW_CONTACT, @@ -235,10 +236,14 @@ const processTrack = async (message, destination) => { const batchIdentify = (arrayChunksIdentify, batchedResponseList, batchOperation) => { // list of chunks [ [..], [..] ] + const { destinationId } = arrayChunksIdentify[0][0].destination; arrayChunksIdentify.forEach((chunk) => { const identifyResponseList = []; const metadata = []; - + // add metric for batch size + stats.gauge('hs_batch_size', chunk.length, { + destination_id: destinationId, + }); // extracting message, destination value // from the first event in a batch const { message, destination } = chunk[0]; diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js index 9eed244af4..fe05d45fbb 100644 --- a/src/v0/destinations/hs/transform.js +++ b/src/v0/destinations/hs/transform.js @@ -1,7 +1,11 @@ const get = require('get-value'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); const { EventType } = require('../../../constants'); -const { handleRtTfSingleEventError, getDestinationExternalIDInfoForRetl } = require('../../util'); +const { + handleRtTfSingleEventError, + getDestinationExternalIDInfoForRetl, + groupEventsByType: batchEventsInOrder, +} = require('../../util'); const { API_VERSION } = require('./config'); const { processLegacyIdentify, @@ -63,19 +67,17 @@ const process = async (event) => { } return processSingleMessage(events[0].message, events[0].destination); }; - -// we are batching by default at routerTransform -const processRouterDest = async (inputs, reqMetadata) => { +const processBatchRouter = async (inputs, reqMetadata) => { let tempInputs = inputs; - - const successRespList = []; - const errorRespList = []; // using the first destination config for transforming the batch const { destination } = tempInputs[0]; let propertyMap; const mappedToDestination = get(tempInputs[0].message, MappedToDestinationKey); const { objectType } = getDestinationExternalIDInfoForRetl(tempInputs[0].message, 'HS'); - + const successRespList = []; + const errorRespList = []; + // batch implementation + let batchedResponseList = []; try { if (mappedToDestination && GENERIC_TRUE_VALUES.includes(mappedToDestination?.toString())) { // skip splitting the batches to inserts and updates if object it is an association @@ -95,11 +97,16 @@ const processRouterDest = async (inputs, reqMetadata) => { } } catch (error) { // Any error thrown from the above try block applies to all the events - return tempInputs.map((input) => handleRtTfSingleEventError(input, error, reqMetadata)); + return { + batchedResponseList, + errorRespList: tempInputs.map((input) => + handleRtTfSingleEventError(input, error, reqMetadata), + ), + }; } await Promise.all( - tempInputs.map(async (input) => { + inputs.map(async (input) => { try { if (input.message.statusCode) { // already transformed event @@ -137,8 +144,6 @@ const processRouterDest = async (inputs, reqMetadata) => { }), ); - // batch implementation - let batchedResponseList = []; if (successRespList.length > 0) { if (destination.Config.apiVersion === API_VERSION.v3) { batchedResponseList = batchEvents(successRespList); @@ -146,6 +151,24 @@ const processRouterDest = async (inputs, reqMetadata) => { batchedResponseList = legacyBatchEvents(successRespList); } } + return { batchedResponseList, errorRespList }; +}; +// we are batching by default at routerTransform +const processRouterDest = async (inputs, reqMetadata) => { + const tempNewInputs = batchEventsInOrder(inputs); + const batchedResponseList = []; + const errorRespList = []; + const promises = tempNewInputs.map(async (inputEvents) => { + const response = await processBatchRouter(inputEvents, reqMetadata); + return response; + }); + + const results = await Promise.all(promises); + + results.forEach((response) => { + errorRespList.push(...response.errorRespList); + batchedResponseList.push(...response.batchedResponseList); + }); return [...batchedResponseList, ...errorRespList]; }; diff --git a/test/integrations/destinations/hs/router/config.ts b/test/integrations/destinations/hs/router/config.ts new file mode 100644 index 0000000000..89a0c13d1a --- /dev/null +++ b/test/integrations/destinations/hs/router/config.ts @@ -0,0 +1,57 @@ +export const destination = { + Config: { + accessToken: 'dummy-access-token', + hubID: 'dummy-hubId', + authorizationType: 'newPrivateAppApi', + apiVersion: 'newApi', + lookupField: 'email', + hubspotEvents: [ + { + rsEventName: 'Purchase', + hubspotEventName: 'pedummy-hubId_rs_hub_test', + eventProperties: [ + { + from: 'Revenue', + to: 'value', + }, + { + from: 'Price', + to: 'cost', + }, + ], + }, + { + rsEventName: 'Purchase2', + hubspotEventName: 'pedummy-hubId_rs_hub_test', + eventProperties: [ + { + from: 'Revenue', + to: 'value', + }, + { + from: 'Price', + to: 'cost', + }, + ], + }, + { + rsEventName: 'Order Complete', + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + eventProperties: [ + { + from: 'firstName', + to: 'first_name', + }, + { + from: 'lastName', + to: 'last_name', + }, + ], + }, + ], + }, + destinationDefinition: { + id: '1aIXqM806xAVm92nx07YwKbRrO9', + }, + transformations: [], +}; diff --git a/test/integrations/destinations/hs/router/data.ts b/test/integrations/destinations/hs/router/data.ts index ab3ca8cba8..e12efef4d0 100644 --- a/test/integrations/destinations/hs/router/data.ts +++ b/test/integrations/destinations/hs/router/data.ts @@ -1,3 +1,4 @@ +import { destination } from './config'; export const data = [ { name: 'hs', @@ -100,6 +101,235 @@ export const data = [ }, }, }, + { + name: 'hs', + description: 'legacy router retl tests', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + channel: 'web', + context: { + mappedToDestination: true, + externalId: [ + { identifierType: 'email', id: 'testhubspot2@email.com', type: 'HS-lead' }, + ], + sources: { + job_id: '24c5HJxHomh6YCngEOCgjS5r1KX/Syncher', + task_id: 'vw_rs_mailchimp_mocked_hg_data', + version: 'v1.8.1', + batch_id: 'f252c69d-c40d-450e-bcd2-2cf26cb62762', + job_run_id: 'c8el40l6e87v0c4hkbl0', + task_run_id: 'c8el40l6e87v0c4hkblg', + }, + }, + type: 'identify', + traits: { firstname: 'Test Hubspot', anonymousId: '12345', country: 'India' }, + messageId: '50360b9c-ea8d-409c-b672-c9230f91cce5', + originalTimestamp: '2019-10-15T09:35:31.288Z', + anonymousId: '00000000000000000000000000', + userId: '12345', + integrations: { All: true }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + Config: { apiKey: 'dummy-apikey', hubID: 'dummy-hubId' }, + secretConfig: {}, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + name: 'Hubspot', + enabled: true, + workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', + deleted: false, + createdAt: '2020-12-30T08:39:32.005Z', + updatedAt: '2021-02-03T16:22:31.374Z', + destinationDefinition: { + id: '1aIXqM806xAVm92nx07YwKbRrO9', + name: 'HS', + displayName: 'Hubspot', + createdAt: '2020-04-09T09:24:31.794Z', + updatedAt: '2021-01-11T11:03:28.103Z', + }, + transformations: [], + isConnectionEnabled: true, + isProcessorEnabled: true, + }, + metadata: { jobId: 2, userId: 'u1' }, + }, + { + message: { + channel: 'web', + context: { + mappedToDestination: true, + externalId: [ + { identifierType: 'email', id: 'testhubspot@email.com', type: 'HS-lead' }, + ], + sources: { + job_id: '24c5HJxHomh6YCngEOCgjS5r1KX/Syncher', + task_id: 'vw_rs_mailchimp_mocked_hg_data', + version: 'v1.8.1', + batch_id: 'f252c69d-c40d-450e-bcd2-2cf26cb62762', + job_run_id: 'c8el40l6e87v0c4hkbl0', + task_run_id: 'c8el40l6e87v0c4hkblg', + }, + }, + type: 'identify', + traits: { firstname: 'Test Hubspot 1', anonymousId: '123451', country: 'India 1' }, + messageId: '50360b9c-ea8d-409c-b672-c9230f91cce5', + originalTimestamp: '2019-10-15T09:35:31.288Z', + anonymousId: '00000000000000000000000000', + userId: '12345', + integrations: { All: true }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + Config: { apiKey: 'dummy-apikey', hubID: 'dummy-hubId' }, + secretConfig: {}, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + name: 'Hubspot', + enabled: true, + workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', + deleted: false, + createdAt: '2020-12-30T08:39:32.005Z', + updatedAt: '2021-02-03T16:22:31.374Z', + destinationDefinition: { + id: '1aIXqM806xAVm92nx07YwKbRrO9', + name: 'HS', + displayName: 'Hubspot', + createdAt: '2020-04-09T09:24:31.794Z', + updatedAt: '2021-01-11T11:03:28.103Z', + }, + transformations: [], + isConnectionEnabled: true, + isProcessorEnabled: true, + }, + metadata: { jobId: 3, userId: 'u1' }, + }, + ], + destType: 'hs', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.hubapi.com/crm/v3/objects/lead/batch/create', + headers: { 'Content-Type': 'application/json' }, + params: { hapikey: 'dummy-apikey' }, + body: { + JSON: { + inputs: [ + { + properties: { + firstname: 'Test Hubspot 1', + anonymousId: '123451', + country: 'India 1', + email: 'testhubspot@email.com', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [{ jobId: 3, userId: 'u1' }], + batched: true, + statusCode: 200, + destination: { + Config: { apiKey: 'dummy-apikey', hubID: 'dummy-hubId' }, + secretConfig: {}, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + name: 'Hubspot', + enabled: true, + workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', + deleted: false, + createdAt: '2020-12-30T08:39:32.005Z', + updatedAt: '2021-02-03T16:22:31.374Z', + destinationDefinition: { + id: '1aIXqM806xAVm92nx07YwKbRrO9', + name: 'HS', + displayName: 'Hubspot', + createdAt: '2020-04-09T09:24:31.794Z', + updatedAt: '2021-01-11T11:03:28.103Z', + }, + transformations: [], + isConnectionEnabled: true, + isProcessorEnabled: true, + }, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.hubapi.com/crm/v3/objects/lead/batch/update', + headers: { 'Content-Type': 'application/json' }, + params: { hapikey: 'dummy-apikey' }, + body: { + JSON: { + inputs: [ + { + properties: { + firstname: 'Test Hubspot', + anonymousId: '12345', + country: 'India', + email: 'testhubspot2@email.com', + }, + id: '103605', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [{ jobId: 2, userId: 'u1' }], + batched: true, + statusCode: 200, + destination: { + Config: { apiKey: 'dummy-apikey', hubID: 'dummy-hubId' }, + secretConfig: {}, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + name: 'Hubspot', + enabled: true, + workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', + deleted: false, + createdAt: '2020-12-30T08:39:32.005Z', + updatedAt: '2021-02-03T16:22:31.374Z', + destinationDefinition: { + id: '1aIXqM806xAVm92nx07YwKbRrO9', + name: 'HS', + displayName: 'Hubspot', + createdAt: '2020-04-09T09:24:31.794Z', + updatedAt: '2021-01-11T11:03:28.103Z', + }, + transformations: [], + isConnectionEnabled: true, + isProcessorEnabled: true, + }, + }, + ], + }, + }, + }, + }, { name: 'hs', description: 'legacy router tests', @@ -299,7 +529,7 @@ export const data = [ }, metadata: { jobId: 4, userId: 'u1' }, destination: { - Config: { apiKey: 'rate-limit-id', hubID: 'dummy-hubId' }, + Config: { apiKey: 'dummy-apikey', hubID: 'dummy-hubId' }, secretConfig: {}, ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', name: 'Hubspot', @@ -343,18 +573,14 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"email":"testhubspot3@email.com","properties":[{"property":"firstname","value":"Test Hubspot3"}]},{"email":"testhubspot1@email.com","properties":[{"property":"firstname","value":"Test Hubspot1"}]},{"email":"testhubspot4@email.com","properties":[{"property":"firstname","value":"Test Hubspot4"}]}]', + '[{"email":"testhubspot1@email.com","properties":[{"property":"firstname","value":"Test Hubspot1"}]}]', }, XML: {}, FORM: {}, }, files: {}, }, - metadata: [ - { jobId: 3, userId: 'u1' }, - { jobId: 1, userId: 'u1' }, - { jobId: 4, userId: 'u1' }, - ], + metadata: [{ jobId: 1, userId: 'u1' }], batched: true, statusCode: 200, destination: { @@ -420,152 +646,20 @@ export const data = [ isProcessorEnabled: true, }, }, - ], - }, - }, - }, - }, - { - name: 'hs', - description: 'legacy router retl tests', - feature: 'router', - module: 'destination', - version: 'v0', - input: { - request: { - body: { - input: [ - { - message: { - channel: 'web', - context: { - mappedToDestination: true, - externalId: [ - { identifierType: 'email', id: 'testhubspot2@email.com', type: 'HS-lead' }, - ], - sources: { - job_id: '24c5HJxHomh6YCngEOCgjS5r1KX/Syncher', - task_id: 'vw_rs_mailchimp_mocked_hg_data', - version: 'v1.8.1', - batch_id: 'f252c69d-c40d-450e-bcd2-2cf26cb62762', - job_run_id: 'c8el40l6e87v0c4hkbl0', - task_run_id: 'c8el40l6e87v0c4hkblg', - }, - }, - type: 'identify', - traits: { firstname: 'Test Hubspot', anonymousId: '12345', country: 'India' }, - messageId: '50360b9c-ea8d-409c-b672-c9230f91cce5', - originalTimestamp: '2019-10-15T09:35:31.288Z', - anonymousId: '00000000000000000000000000', - userId: '12345', - integrations: { All: true }, - sentAt: '2019-10-14T09:03:22.563Z', - }, - destination: { - Config: { apiKey: 'dummy-apikey', hubID: 'dummy-hubId' }, - secretConfig: {}, - ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', - name: 'Hubspot', - enabled: true, - workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', - deleted: false, - createdAt: '2020-12-30T08:39:32.005Z', - updatedAt: '2021-02-03T16:22:31.374Z', - destinationDefinition: { - id: '1aIXqM806xAVm92nx07YwKbRrO9', - name: 'HS', - displayName: 'Hubspot', - createdAt: '2020-04-09T09:24:31.794Z', - updatedAt: '2021-01-11T11:03:28.103Z', - }, - transformations: [], - isConnectionEnabled: true, - isProcessorEnabled: true, - }, - metadata: { jobId: 2, userId: 'u1' }, - }, - { - message: { - channel: 'web', - context: { - mappedToDestination: true, - externalId: [ - { identifierType: 'email', id: 'testhubspot@email.com', type: 'HS-lead' }, - ], - sources: { - job_id: '24c5HJxHomh6YCngEOCgjS5r1KX/Syncher', - task_id: 'vw_rs_mailchimp_mocked_hg_data', - version: 'v1.8.1', - batch_id: 'f252c69d-c40d-450e-bcd2-2cf26cb62762', - job_run_id: 'c8el40l6e87v0c4hkbl0', - task_run_id: 'c8el40l6e87v0c4hkblg', - }, - }, - type: 'identify', - traits: { firstname: 'Test Hubspot 1', anonymousId: '123451', country: 'India 1' }, - messageId: '50360b9c-ea8d-409c-b672-c9230f91cce5', - originalTimestamp: '2019-10-15T09:35:31.288Z', - anonymousId: '00000000000000000000000000', - userId: '12345', - integrations: { All: true }, - sentAt: '2019-10-14T09:03:22.563Z', - }, - destination: { - Config: { apiKey: 'dummy-apikey', hubID: 'dummy-hubId' }, - secretConfig: {}, - ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', - name: 'Hubspot', - enabled: true, - workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', - deleted: false, - createdAt: '2020-12-30T08:39:32.005Z', - updatedAt: '2021-02-03T16:22:31.374Z', - destinationDefinition: { - id: '1aIXqM806xAVm92nx07YwKbRrO9', - name: 'HS', - displayName: 'Hubspot', - createdAt: '2020-04-09T09:24:31.794Z', - updatedAt: '2021-01-11T11:03:28.103Z', - }, - transformations: [], - isConnectionEnabled: true, - isProcessorEnabled: true, - }, - metadata: { jobId: 3, userId: 'u1' }, - }, - ], - destType: 'hs', - }, - method: 'POST', - }, - }, - output: { - response: { - status: 200, - body: { - output: [ { batchedRequest: { version: '1', type: 'REST', method: 'POST', - endpoint: 'https://api.hubapi.com/crm/v3/objects/lead/batch/create', + endpoint: 'https://api.hubapi.com/contacts/v1/contact/batch/', headers: { 'Content-Type': 'application/json' }, params: { hapikey: 'dummy-apikey' }, body: { - JSON: { - inputs: [ - { - properties: { - firstname: 'Test Hubspot 1', - anonymousId: '123451', - country: 'India 1', - email: 'testhubspot@email.com', - }, - }, - ], + JSON: {}, + JSON_ARRAY: { + batch: + '[{"email":"testhubspot3@email.com","properties":[{"property":"firstname","value":"Test Hubspot3"}]}]', }, - JSON_ARRAY: {}, XML: {}, FORM: {}, }, @@ -601,30 +695,21 @@ export const data = [ version: '1', type: 'REST', method: 'POST', - endpoint: 'https://api.hubapi.com/crm/v3/objects/lead/batch/update', + endpoint: 'https://api.hubapi.com/contacts/v1/contact/batch/', headers: { 'Content-Type': 'application/json' }, params: { hapikey: 'dummy-apikey' }, body: { - JSON: { - inputs: [ - { - properties: { - firstname: 'Test Hubspot', - anonymousId: '12345', - country: 'India', - email: 'testhubspot2@email.com', - }, - id: '103605', - }, - ], + JSON: {}, + JSON_ARRAY: { + batch: + '[{"email":"testhubspot4@email.com","properties":[{"property":"firstname","value":"Test Hubspot4"}]}]', }, - JSON_ARRAY: {}, XML: {}, FORM: {}, }, files: {}, }, - metadata: [{ jobId: 2, userId: 'u1' }], + metadata: [{ jobId: 4, userId: 'u1' }], batched: true, statusCode: 200, destination: { @@ -1280,21 +1365,37 @@ export const data = [ }, { message: { - type: 'track', - traits: {}, + channel: 'web', context: { - externalId: [ - { - id: 'osvaldocostaferreira98@gmail.com', - type: 'HS-contacts', - identifierType: 'email', - }, - ], + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + traits: { + email: 'testhubspot@email.com', + firstname: 'Test Hubspot22', + anonymousId: '4444', + }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-GB', + ip: '0.0.0.0', + os: { name: '', version: '' }, + screen: { density: 2 }, + page: { path: '', referrer: '', search: '', title: '', url: '' }, }, - event: 'Purchase', - properties: { Revenue: 'name1' }, + type: 'identify', + messageId: '50360b9c-ea8d-409c-b672-c9230f91cce5', + originalTimestamp: '2019-10-15T09:35:31.288Z', + anonymousId: '00000000000000000000000000', + userId: '12345', + integrations: { All: true }, + sentAt: '2019-10-14T09:03:22.563Z', }, - metadata: { jobId: 3, userId: 'u1' }, + metadata: { jobId: 4, userId: 'u1' }, destination: { Config: { authorizationType: 'newPrivateAppApi', @@ -1302,7 +1403,7 @@ export const data = [ hubID: 'dummy-hubId', apiKey: 'dummy-apikey', apiVersion: 'newApi', - lookupField: 'lookupField', + lookupField: 'email', hubspotEvents: [ { rsEventName: 'Purchase', @@ -1357,7 +1458,7 @@ export const data = [ }, traits: { email: 'testhubspot@email.com', - firstname: 'Test Hubspot22', + firstname: 'Test Hubspot44', anonymousId: '4444', }, library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, @@ -1377,7 +1478,7 @@ export const data = [ integrations: { All: true }, sentAt: '2019-10-14T09:03:22.563Z', }, - metadata: { jobId: 4, userId: 'u1' }, + metadata: { jobId: 5, userId: 'u1' }, destination: { Config: { authorizationType: 'newPrivateAppApi', @@ -1430,37 +1531,21 @@ export const data = [ }, { message: { - channel: 'web', + type: 'track', + traits: {}, context: { - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.0.0', - }, - traits: { - email: 'testhubspot@email.com', - firstname: 'Test Hubspot44', - anonymousId: '4444', - }, - library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - locale: 'en-GB', - ip: '0.0.0.0', - os: { name: '', version: '' }, - screen: { density: 2 }, - page: { path: '', referrer: '', search: '', title: '', url: '' }, + externalId: [ + { + id: 'osvaldocostaferreira98@gmail.com', + type: 'HS-contacts', + identifierType: 'email', + }, + ], }, - type: 'identify', - messageId: '50360b9c-ea8d-409c-b672-c9230f91cce5', - originalTimestamp: '2019-10-15T09:35:31.288Z', - anonymousId: '00000000000000000000000000', - userId: '12345', - integrations: { All: true }, - sentAt: '2019-10-14T09:03:22.563Z', + event: 'Purchase', + properties: { Revenue: 'name1' }, }, - metadata: { jobId: 5, userId: 'u1' }, + metadata: { jobId: 3, userId: 'u1' }, destination: { Config: { authorizationType: 'newPrivateAppApi', @@ -1468,7 +1553,7 @@ export const data = [ hubID: 'dummy-hubId', apiKey: 'dummy-apikey', apiVersion: 'newApi', - lookupField: 'email', + lookupField: 'lookupField', hubspotEvents: [ { rsEventName: 'Purchase', @@ -1639,7 +1724,7 @@ export const data = [ hubID: 'dummy-hubId', apiKey: 'dummy-apikey', apiVersion: 'newApi', - lookupField: 'email', + lookupField: 'lookupField', hubspotEvents: [ { rsEventName: 'Purchase', @@ -1828,4 +1913,554 @@ export const data = [ }, }, }, + { + name: 'hs', + description: 'router job ordering ', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + channel: 'web', + context: { + traits: { + email: 'testhubspot1@email.com', + firstname: 'Test Hubspot1', + }, + }, + type: 'identify', + userId: 'user1', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + metadata: { + jobId: 1, + userId: 'user1', + }, + destination, + }, + { + message: { + channel: 'web', + context: { + traits: { email: 'user1@a.com' }, + }, + type: 'track', + anonymousId: '', + userId: 'user1', + event: 'purchase', + properties: { + user_actual_role: 'system_admin, system_user', + user_actual_id: 12345, + }, + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + jobId: 2, + userId: 'user1', + }, + destination, + }, + { + message: { + channel: 'web', + context: { traits: { email: 'user2@a.com' } }, + type: 'track', + anonymousId: '', + userId: 'user2', + event: 'purchase2', + properties: { + user_actual_role: 'system_admin_2, system_user', + user_actual_id: 12345, + }, + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + jobId: 3, + userId: 'user2', + }, + destination, + }, + { + message: { + channel: 'web', + context: { + traits: { + email: 'testhubspot2@email.com', + firstname: 'Test Hubspot1', + anonymousId: '1111', + }, + }, + type: 'identify', + anonymousId: '', + userId: 'user2', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + metadata: { + jobId: 4, + userId: 'user2', + }, + destination, + }, + { + message: { + channel: 'web', + context: { traits: { email: 'user3@a.com' } }, + type: 'track', + anonymousId: '', + userId: 'user3', + event: 'purchase', + properties: { + user_actual_role: 'system_admin, system_user', + user_actual_id: 12345, + }, + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + jobId: 5, + userId: 'user3', + }, + destination, + }, + { + message: { + channel: 'web', + context: { + traits: { + email: 'testhubspot3@email.com', + firstname: 'Test Hubspot1', + anonymousId: '1111', + }, + }, + type: 'identify', + anonymousId: '', + userId: 'user3', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + metadata: { + jobId: 6, + userId: 'user3', + }, + destination, + }, + { + message: { + channel: 'web', + context: { traits: { email: 'user4@a.com' } }, + type: 'track', + anonymousId: '', + userId: 'user4', + event: 'purchase', + properties: { + user_actual_role: 'system_admin, system_user', + user_actual_id: 12345, + }, + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + jobId: 7, + userId: 'user4', + }, + destination, + }, + { + message: { + channel: 'web', + context: { + traits: { + email: 'testhubspot4@email.com', + firstname: 'Test Hubspot4', + anonymousId: '1111', + }, + }, + type: 'identify', + anonymousId: '', + userId: 'user4', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + metadata: { + jobId: 8, + userId: 'user4', + }, + destination, + }, + { + message: { + channel: 'web', + context: { + traits: { + email: 'testhubspot5@email.com', + firstname: 'Test Hubspot51', + anonymousId: '1111', + }, + }, + type: 'identify', + anonymousId: '', + userId: 'user5', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + metadata: { + jobId: 9, + userId: 'user5', + }, + destination, + }, + { + message: { + channel: 'web', + context: { traits: { email: 'user5@a.com' } }, + type: 'track', + anonymousId: '', + userId: 'user5', + event: 'purchase', + properties: { + user_actual_role: 'system_admin, system_user', + user_actual_id: 12345, + }, + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + jobId: 10, + userId: 'user5', + }, + destination, + }, + { + message: { + channel: 'web', + context: { + traits: { + email: 'testhubspot5@email.com', + firstname: 'Test Hubspot5', + anonymousId: '1111', + }, + }, + type: 'identify', + anonymousId: '', + userId: 'user5', + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + metadata: { + jobId: 11, + userId: 'user5', + }, + destination, + }, + ], + destType: 'hs', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/create', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + params: {}, + body: { + JSON: { + inputs: [ + { + properties: { + email: 'testhubspot1@email.com', + firstname: 'Test Hubspot1', + }, + }, + { + properties: { + email: 'testhubspot5@email.com', + firstname: 'Test Hubspot51', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 1, + userId: 'user1', + }, + { + jobId: 9, + userId: 'user5', + }, + ], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.hubapi.com/events/v3/send', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + params: {}, + body: { + JSON: { + email: 'user1@a.com', + eventName: 'pedummy-hubId_rs_hub_test', + properties: {}, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 2, + userId: 'user1', + }, + ], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.hubapi.com/events/v3/send', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + params: {}, + body: { + JSON: { + email: 'user2@a.com', + eventName: 'pedummy-hubId_rs_hub_test', + properties: {}, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 3, + userId: 'user2', + }, + ], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.hubapi.com/events/v3/send', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + params: {}, + body: { + JSON: { + email: 'user3@a.com', + eventName: 'pedummy-hubId_rs_hub_test', + properties: {}, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 5, + userId: 'user3', + }, + ], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.hubapi.com/events/v3/send', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + params: {}, + body: { + JSON: { + email: 'user4@a.com', + eventName: 'pedummy-hubId_rs_hub_test', + properties: {}, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 7, + userId: 'user4', + }, + ], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.hubapi.com/events/v3/send', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + params: {}, + body: { + JSON: { + email: 'user5@a.com', + eventName: 'pedummy-hubId_rs_hub_test', + properties: {}, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 10, + userId: 'user5', + }, + ], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/create', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + params: {}, + body: { + JSON: { + inputs: [ + { + properties: { + email: 'testhubspot2@email.com', + firstname: 'Test Hubspot1', + }, + }, + { + properties: { + email: 'testhubspot3@email.com', + firstname: 'Test Hubspot1', + }, + }, + { + properties: { + email: 'testhubspot4@email.com', + firstname: 'Test Hubspot4', + }, + }, + { + properties: { + email: 'testhubspot5@email.com', + firstname: 'Test Hubspot5', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 4, + userId: 'user2', + }, + { + jobId: 6, + userId: 'user3', + }, + { + jobId: 8, + userId: 'user4', + }, + { + jobId: 11, + userId: 'user5', + }, + ], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + }, ]; From 1bef2126a75324598c2af0ecaffcf582f038af11 Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:57:08 +0530 Subject: [PATCH 03/28] fix: add validation for type in google pubsub (#3578) --- src/v0/destinations/googlepubsub/util.js | 5 ++ .../googlepubsub/processor/data.ts | 68 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/src/v0/destinations/googlepubsub/util.js b/src/v0/destinations/googlepubsub/util.js index 8af71ac25f..a163356ed0 100644 --- a/src/v0/destinations/googlepubsub/util.js +++ b/src/v0/destinations/googlepubsub/util.js @@ -1,3 +1,4 @@ +const { ConfigurationAuthError } = require('@rudderstack/integrations-lib'); const { getHashFromArray, getValueFromMessage, @@ -18,6 +19,10 @@ const getTopic = (event) => { const { eventToTopicMap } = destination.Config; const hashMap = getHashFromArray(eventToTopicMap, 'from', 'to'); + if (!message.type) { + throw new ConfigurationAuthError('type is required for event'); + } + return ( (message.event ? hashMap[message.event.toLowerCase()] : null) || hashMap[message.type.toLowerCase()] || diff --git a/test/integrations/destinations/googlepubsub/processor/data.ts b/test/integrations/destinations/googlepubsub/processor/data.ts index 0dffa57839..6746e5b765 100644 --- a/test/integrations/destinations/googlepubsub/processor/data.ts +++ b/test/integrations/destinations/googlepubsub/processor/data.ts @@ -1314,4 +1314,72 @@ export const data = [ }, }, }, + { + name: 'googlepubsub', + description: 'Test 13', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + sentAt: '2020-08-28T15:11:56.167Z', + category: 'Food', + messageId: + 'node-cfc5fb7ec83b82bc29e16336a11331e2-0ba97212-0f6e-44cd-a0f1-c20b8b7a7cba', + anonymousId: 'abcdeeeeeeeexxxx111', + originalTimestamp: '2020-08-28T15:11:56.162Z', + name: 'Pizza', + _metadata: { + nodeVersion: '10.22.0', + }, + }, + destination: { + Config: { + credentials: 'abc', + eventToTopicMap: [ + { + from: 'track', + to: 'Test-Topic', + }, + { + from: '*', + to: 'test', + }, + ], + eventToAttributesMap: [ + { + from: 'track', + to: 'properties.nestedObject.this', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: 'type is required for event', + statTags: { + destType: 'GOOGLEPUBSUB', + errorCategory: 'dataValidation', + errorType: 'configuration', + feature: 'processor', + implementation: 'native', + meta: 'accessTokenExpired', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + }, ]; From 6304abb2346f331b78e927b73e7c2ca17e94f4cf Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Fri, 19 Jul 2024 18:21:40 +0530 Subject: [PATCH 04/28] feat: introduces new user fields in titkok ads (#3575) * feat: introduces new user fields in titkok ads * fix: add hashing 256 to first name , last name and zip code * chore: address comments * chore: fix lint --- .../tiktok_ads/data/TikTokTrackV2.json | 36 +++++++++++++++++++ src/v0/destinations/tiktok_ads/util.js | 1 + src/v0/util/index.js | 5 +++ .../destinations/tiktok_ads/processor/data.ts | 17 +++++++++ 4 files changed, 59 insertions(+) diff --git a/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json b/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json index 2910f1b44c..ec66cf9ba2 100644 --- a/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json +++ b/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json @@ -126,5 +126,41 @@ "destKey": "user.user_agent", "sourceKeys": ["properties.context.user.userAgent", "context.userAgent"], "required": false + }, + { + "destKey": "user.first_name", + "sourceKeys": "firstName", + "metadata": { + "type": ["trim", "toLower", "hashToSha256"] + }, + "sourceFromGenericMap": true + }, + { + "destKey": "user.last_name", + "sourceKeys": "lastName", + "metadata": { + "type": ["trim", "toLower", "hashToSha256"] + }, + "sourceFromGenericMap": true + }, + { + "destKey": "user.city", + "sourceKeys": ["context.traits.city", "context.traits.address.city", "properties.city"] + }, + { + "destKey": "user.country", + "sourceKeys": ["context.traits.country", "context.traits.address.country", "properties.country"] + }, + { + "destKey": "user.state", + "sourceKeys": ["context.traits.address.state", "context.traits.state", "properties.state"] + }, + { + "destKey": "user.zip_code", + "sourceKeys": "zipcode", + "sourceFromGenericMap": true, + "metadata": { + "type": ["removeSpacesAndDashes", "hashToSha256"] + } } ] diff --git a/src/v0/destinations/tiktok_ads/util.js b/src/v0/destinations/tiktok_ads/util.js index 3d86ac69b7..5f86193531 100644 --- a/src/v0/destinations/tiktok_ads/util.js +++ b/src/v0/destinations/tiktok_ads/util.js @@ -23,6 +23,7 @@ const getContents = (message, getContentType = true) => { price: product.price, quantity: product.quantity, description: product.description, + brand: product.brand, }; contents.push(removeUndefinedAndNullValues(singleProduct)); }); diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 12b8d4dd7e..f30ccbbfd1 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -844,6 +844,11 @@ function formatValues(formattedVal, formattingType, typeFormat, integrationsObj) curFormattedVal = formattedVal.trim(); } }, + removeSpacesAndDashes: () => { + if (typeof formattedVal === 'string') { + curFormattedVal = formattedVal.replace(/ /g, '').replace(/-/g, ''); + } + }, }; if (formattingType in formattingFunctions) { diff --git a/test/integrations/destinations/tiktok_ads/processor/data.ts b/test/integrations/destinations/tiktok_ads/processor/data.ts index f459e68681..e6fe407381 100644 --- a/test/integrations/destinations/tiktok_ads/processor/data.ts +++ b/test/integrations/destinations/tiktok_ads/processor/data.ts @@ -4619,6 +4619,7 @@ export const data = [ content_name: 'Monopoly', price: 14, quantity: 1, + brand: 'brand_name', }, { content_type: 'product_group', @@ -6354,6 +6355,12 @@ export const data = [ context: { traits: { email: 'abc@xyz.com', + firstName: ' test', + lastName: 'user ', + country: 'dummycountry', + city: 'dummycity', + state: 'dummystate', + zip: ' US - 1234-', }, page: { url: 'http://demo.mywebsite.com/purchase', @@ -6474,6 +6481,7 @@ export const data = [ { price: 14, quantity: 1, + brand: 'adidas', content_category: 'Games', content_id: '123', content_name: 'Monopoly', @@ -6493,6 +6501,15 @@ export const data = [ referrer: 'http://demo.mywebsite.com', }, user: { + first_name: + '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', + last_name: + '04f8996da763b7a969b1028ee3007569eaf3a635486ddab211d512c85b9df8fb', + country: 'dummycountry', + zip_code: + '2a29b91aca86ff48c3defb1bbc2d33bf1da5ee5a235391322478fe7ff4503b77', + city: 'dummycity', + state: 'dummystate', email: 'ee278943de84e5d6243578ee1a1057bcce0e50daad9755f45dfa64b60b13bc5d', external_id: [ '3e0c7a51acd326b87f29596e38c22cbeb732df37bc5c8f5f524c14b55d3472db', From fbcdcd609888150efa0da33eec60a4cc7b436d06 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Mon, 22 Jul 2024 10:40:35 +0530 Subject: [PATCH 05/28] feat: onboard cordial destination (#3581) * feat: onboard cordial destination * fix: remove email optin mapping --- src/cdk/v2/destinations/cordial/config.js | 34 +++ .../cordial/data/CordialIdentifyConfig.json | 11 + .../cordial/data/CordialTrackConfig.json | 21 ++ .../v2/destinations/cordial/procWorkflow.yaml | 106 +++++++++ .../v2/destinations/cordial/rtWorkflow.yaml | 31 +++ src/cdk/v2/destinations/cordial/utils.js | 30 +++ src/constants/destinationCanonicalNames.js | 1 + src/features.json | 3 +- src/v0/destinations/sendinblue/transform.js | 2 +- src/v0/destinations/sendinblue/util.js | 15 -- src/v0/util/index.js | 15 ++ .../destinations/cordial/common.ts | 90 ++++++++ .../destinations/cordial/network.ts | 71 ++++++ .../destinations/cordial/processor/data.ts | 4 + .../cordial/processor/identify.ts | 205 ++++++++++++++++++ .../destinations/cordial/processor/track.ts | 125 +++++++++++ .../cordial/processor/validation.ts | 50 +++++ .../destinations/cordial/router/data.ts | 134 ++++++++++++ 18 files changed, 931 insertions(+), 17 deletions(-) create mode 100644 src/cdk/v2/destinations/cordial/config.js create mode 100644 src/cdk/v2/destinations/cordial/data/CordialIdentifyConfig.json create mode 100644 src/cdk/v2/destinations/cordial/data/CordialTrackConfig.json create mode 100644 src/cdk/v2/destinations/cordial/procWorkflow.yaml create mode 100644 src/cdk/v2/destinations/cordial/rtWorkflow.yaml create mode 100644 src/cdk/v2/destinations/cordial/utils.js create mode 100644 test/integrations/destinations/cordial/common.ts create mode 100644 test/integrations/destinations/cordial/network.ts create mode 100644 test/integrations/destinations/cordial/processor/data.ts create mode 100644 test/integrations/destinations/cordial/processor/identify.ts create mode 100644 test/integrations/destinations/cordial/processor/track.ts create mode 100644 test/integrations/destinations/cordial/processor/validation.ts create mode 100644 test/integrations/destinations/cordial/router/data.ts diff --git a/src/cdk/v2/destinations/cordial/config.js b/src/cdk/v2/destinations/cordial/config.js new file mode 100644 index 0000000000..74362eb442 --- /dev/null +++ b/src/cdk/v2/destinations/cordial/config.js @@ -0,0 +1,34 @@ +const { getMappingConfig } = require('../../../../v0/util'); + +const getCreateContactEndpoint = (config) => `${config.apiBaseUrl}/v2/contacts`; +const getUpdateContactEndpoint = (config, contactId, email) => { + if (contactId) { + return `${config.apiBaseUrl}/v2/contacts/${contactId}`; + } + return `${config.apiBaseUrl}/v2/contacts/email:${email}`; +}; + +const getEventsEndpoint = (config) => `${config.apiBaseUrl}/v2/contactactivities`; +const getContactEndpoint = getUpdateContactEndpoint; + +const CONFIG_CATEGORIES = { + IDENTIFY: { + name: 'CordialIdentifyConfig', + type: 'identify', + }, + TRACK: { + name: 'CordialTrackConfig', + type: 'track', + }, +}; + +const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); + +module.exports = { + IDENTIFY_CONFIG: MAPPING_CONFIG[CONFIG_CATEGORIES.IDENTIFY.name], + TRACK_CONFIG: MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK.name], + getCreateContactEndpoint, + getUpdateContactEndpoint, + getEventsEndpoint, + getContactEndpoint, +}; diff --git a/src/cdk/v2/destinations/cordial/data/CordialIdentifyConfig.json b/src/cdk/v2/destinations/cordial/data/CordialIdentifyConfig.json new file mode 100644 index 0000000000..71d6cf1e9e --- /dev/null +++ b/src/cdk/v2/destinations/cordial/data/CordialIdentifyConfig.json @@ -0,0 +1,11 @@ +[ + { + "destKey": "channels.email.address", + "sourceKeys": "emailOnly", + "sourceFromGenericMap": true + }, + { + "destKey": "channels.email.subscribeStatus", + "sourceKeys": ["traits.subscribeStatus", "context.traits.subscribeStatus"] + } +] diff --git a/src/cdk/v2/destinations/cordial/data/CordialTrackConfig.json b/src/cdk/v2/destinations/cordial/data/CordialTrackConfig.json new file mode 100644 index 0000000000..ddf74ce6b1 --- /dev/null +++ b/src/cdk/v2/destinations/cordial/data/CordialTrackConfig.json @@ -0,0 +1,21 @@ +[ + { + "destKey": "email", + "sourceKeys": "emailOnly", + "sourceFromGenericMap": true + }, + { + "destKey": "a", + "sourceKeys": "event", + "required": true + }, + { + "destKey": "properties", + "sourceKeys": "properties" + }, + { + "destKey": "ats", + "sourceKeys": "timestamp", + "sourceFromGenericMap": true + } +] diff --git a/src/cdk/v2/destinations/cordial/procWorkflow.yaml b/src/cdk/v2/destinations/cordial/procWorkflow.yaml new file mode 100644 index 0000000000..2c765afeb5 --- /dev/null +++ b/src/cdk/v2/destinations/cordial/procWorkflow.yaml @@ -0,0 +1,106 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + exportAll: true + - name: getHashFromArray + path: ../../../../v0/util + - name: getIntegrationsObj + path: ../../../../v0/util + - name: getDestinationExternalID + path: ../../../../v0/util + - name: removeUndefinedAndNullValues + path: ../../../../v0/util + - name: defaultRequestConfig + path: ../../../../v0/util + - name: base64Convertor + path: ../../../../v0/util + - name: constructPayload + path: ../../../../v0/util + - name: removeEmptyKey + path: ../../../../v0/util + - name: CommonUtils + path: ../../../../util/common + - path: ./utils + - path: ./config + +steps: + - name: checkIfProcessed + condition: .message.statusCode + template: | + $.batchMode ? .message.body.JSON : .message + onComplete: return + + - name: messageType + template: | + $.context.messageType = .message.type.toLowerCase(); + + - name: validateInput + template: | + let messageType = $.context.messageType; + $.assert(messageType, "message Type is not present. Aborting"); + $.assert(messageType in {{$.EventType.([.TRACK, .IDENTIFY])}}, "message type " + messageType + " is not supported"); + $.assertConfig(.destination.Config.apiKey, "API Key is not present. Aborting"); + $.assertConfig(.destination.Config.apiBaseUrl, "API Base URl is not present. Aborting"); + + - name: getContactId + template: | + $.getDestinationExternalID(.message,'cordialContactId'); + + - name: getContactEmail + template: | + .message.().({{{{$.getGenericPaths("email")}}}};); + + - name: buildIdentifyPayload + condition: $.context.messageType in [{{$.EventType.IDENTIFY}}] + steps: + - name: checkIfContactExists + template: | + $.checkIfContactExists(.destination.Config, $.outputs.getContactId, $.outputs.getContactEmail); + + - name: buildPayload + template: | + const integrationObj = $.getIntegrationsObj(.message, "cordial"); + let payload = $.constructPayload(.message, $.IDENTIFY_CONFIG); + const traits = .message.().( + {{{{$.getGenericPaths("traits")}}}}; + ); + payload = {...payload, ...traits}; + payload = payload{~["email", "subscribeStatus"]}; + $.context.payload = payload; + + - name: createContact + condition: $.outputs.buildIdentifyPayload.checkIfContactExists === false + template: | + $.context.endpoint = $.getCreateContactEndpoint(.destination.Config); + $.context.method = "POST"; + + - name: updateContact + condition: $.outputs.buildIdentifyPayload.checkIfContactExists === true + template: | + $.context.endpoint = $.getUpdateContactEndpoint( + .destination.Config, $.outputs.getContactId, $.outputs.getContactEmail); + $.context.method = "PUT"; + + - name: buildTrackPayload + condition: $.context.messageType in [{{$.EventType.TRACK}}] + template: | + let payload = $.constructPayload(.message, $.TRACK_CONFIG); + payload.cID = $.outputs.getContactId; + payload.cID ? payload = payload{~["email"]} : payload = payload{~["cID"]}; + $.context.payload = payload; + $.context.endpoint = $.getEventsEndpoint(.destination.Config); + $.context.method = "POST"; + + - name: buildResponseForProcessTransformation + template: | + const payload = $.context.payload; + const response = $.defaultRequestConfig(); + response.body.JSON = $.removeUndefinedAndNullValues($.removeEmptyKey(payload)); + response.endpoint = $.context.endpoint; + response.method = $.context.method; + response.headers = { + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.destination.Config.apiKey + ":") + }; + response; diff --git a/src/cdk/v2/destinations/cordial/rtWorkflow.yaml b/src/cdk/v2/destinations/cordial/rtWorkflow.yaml new file mode 100644 index 0000000000..dd438a911c --- /dev/null +++ b/src/cdk/v2/destinations/cordial/rtWorkflow.yaml @@ -0,0 +1,31 @@ +bindings: + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "batchedRequest": ., + "batched": false, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata[], + "statusCode": 200 + })[] + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + - name: finalPayload + template: | + [...$.outputs.successfulEvents, ...$.outputs.failedEvents] diff --git a/src/cdk/v2/destinations/cordial/utils.js b/src/cdk/v2/destinations/cordial/utils.js new file mode 100644 index 0000000000..1d0ab5789e --- /dev/null +++ b/src/cdk/v2/destinations/cordial/utils.js @@ -0,0 +1,30 @@ +const { getContactEndpoint, destType } = require('./config'); +const { handleHttpRequest } = require('../../../../adapters/network'); + +const checkIfContactExists = async (config, contactId, email) => { + const basicAuth = Buffer.from(`${config.apiKey}:`).toString('base64'); + const endpoint = getContactEndpoint(config, contactId, email); + const { processedResponse } = await handleHttpRequest( + 'get', + endpoint, + { + headers: { + Authorization: `Basic ${basicAuth}`, + }, + }, + { + destType, + feature: 'transformation', + requestMethod: 'GET', + endpointPath: contactId ? '/contacts' : '/contacts/email', + module: 'router', + }, + ); + + // eslint-disable-next-line no-underscore-dangle + return processedResponse.status === 200 && !!processedResponse.response?._id; +}; + +module.exports = { + checkIfContactExists, +}; diff --git a/src/constants/destinationCanonicalNames.js b/src/constants/destinationCanonicalNames.js index 915ac50b26..58bf35539a 100644 --- a/src/constants/destinationCanonicalNames.js +++ b/src/constants/destinationCanonicalNames.js @@ -175,6 +175,7 @@ const DestCanonicalNames = { ], emarsys: ['EMARSYS', 'Emarsys', 'emarsys'], wunderkind: ['wunderkind', 'Wunderkind', 'WUNDERKIND'], + cordial: ['cordial', 'Cordial', 'CORDIAL'], }; module.exports = { DestHandlerMap, DestCanonicalNames }; diff --git a/src/features.json b/src/features.json index ade8c90439..78737193a8 100644 --- a/src/features.json +++ b/src/features.json @@ -74,7 +74,8 @@ "EMARSYS": true, "KODDI": true, "WUNDERKIND": true, - "CLICKSEND": true + "CLICKSEND": true, + "CORDIAL": true }, "regulations": [ "BRAZE", diff --git a/src/v0/destinations/sendinblue/transform.js b/src/v0/destinations/sendinblue/transform.js index 7663672f12..9514359d02 100644 --- a/src/v0/destinations/sendinblue/transform.js +++ b/src/v0/destinations/sendinblue/transform.js @@ -13,6 +13,7 @@ const { defaultPutRequestConfig, getIntegrationsObj, ErrorMessage, + removeEmptyKey, } = require('../../util'); const { CONFIG_CATEGORIES, MAPPING_CONFIG, getUnlinkContactEndpoint } = require('./config'); const { @@ -21,7 +22,6 @@ const { validateEmailAndPhone, checkIfContactExists, prepareHeader, - removeEmptyKey, transformUserTraits, prepareTrackEventData, getListIds, diff --git a/src/v0/destinations/sendinblue/util.js b/src/v0/destinations/sendinblue/util.js index 9fded8e493..e4862ccc39 100644 --- a/src/v0/destinations/sendinblue/util.js +++ b/src/v0/destinations/sendinblue/util.js @@ -90,20 +90,6 @@ const checkIfContactExists = async (identifier, apiKey) => { return false; }; -/** - * Function to remove empty key ("") from payload - * @param {*} payload {"key1":"a","":{"id":1}} - * @returns // {"key1":"a"} - */ -const removeEmptyKey = (payload) => { - const rawPayload = payload; - const key = ''; - if (Object.prototype.hasOwnProperty.call(rawPayload, key)) { - delete rawPayload['']; - } - return rawPayload; -}; - /** * Function to remove duplicate traits from user traits * @param {*} userTraits {"location":"San Francisco","LOCATION":"San Francisco"} @@ -171,7 +157,6 @@ module.exports = { validateEmailAndPhone, checkIfContactExists, prepareHeader, - removeEmptyKey, transformUserTraits, prepareTrackEventData, getListIds, diff --git a/src/v0/util/index.js b/src/v0/util/index.js index f30ccbbfd1..dc06bcc43a 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -171,6 +171,20 @@ const isDefinedNotNullNotEmpty = (value) => const removeUndefinedNullEmptyExclBoolInt = (obj) => lodash.pickBy(obj, isDefinedNotNullNotEmpty); +/** + * Function to remove empty key ("") from payload + * @param {*} payload {"key1":"a","":{"id":1}} + * @returns // {"key1":"a"} + */ +const removeEmptyKey = (payload) => { + const rawPayload = payload; + const key = ''; + if (Object.prototype.hasOwnProperty.call(rawPayload, key)) { + delete rawPayload['']; + } + return rawPayload; +}; + /** * Recursively removes undefined, null, empty objects, and empty arrays from the given object at all levels. * @param {*} obj @@ -2376,4 +2390,5 @@ module.exports = { removeDuplicateMetadata, combineBatchRequestsWithSameJobIds, validateEventAndLowerCaseConversion, + removeEmptyKey, }; diff --git a/test/integrations/destinations/cordial/common.ts b/test/integrations/destinations/cordial/common.ts new file mode 100644 index 0000000000..1ebd0db34b --- /dev/null +++ b/test/integrations/destinations/cordial/common.ts @@ -0,0 +1,90 @@ +import { Destination } from '../../../../src/types'; + +const destType = 'cordial'; +const destTypeInUpperCase = 'CORDIAL'; +const displayName = 'Cordial'; +const destination: Destination = { + Config: { + apiBaseUrl: 'https://abc.example.com', + apiKey: 'test-api-key', + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', +}; +const traits = { + email: 'johndoe@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + address: { + city: 'New York', + country: 'USA', + pin_code: '123456', + }, + subscribeStatus: 'subscribed', +}; +const endpoint = 'https://abc.example.com/v2/contacts'; +const updateContactEmailEndpoint = 'https://abc.example.com/v2/contacts/email:johndoe@example.com'; +const updateContactIdEndpoint = 'https://abc.example.com/v2/contacts/6690fe3655e334d6270287b5'; +const eventsEndpoint = 'https://abc.example.com/v2/contactactivities'; +const context = { + externalId: [ + { + type: 'cordialContactId', + id: '6690fe3655e334d6270287b5', + }, + ], +}; +const headers = { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC1hcGkta2V5Og==', +}; +const properties = { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + category: 'Games', + name: 'Cones of Dunshire', + brand: 'Wyatt Games', + variant: 'expansion pack', + price: 49.99, + quantity: 5, + coupon: 'PREORDER15', + currency: 'USD', + position: 1, + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.webp', + key1: 'value1', +}; +const processorInstrumentationErrorStatTags = { + destType: destTypeInUpperCase, + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', +}; + +export { + destType, + destination, + traits, + endpoint, + updateContactEmailEndpoint, + updateContactIdEndpoint, + eventsEndpoint, + context, + headers, + properties, + processorInstrumentationErrorStatTags, +}; diff --git a/test/integrations/destinations/cordial/network.ts b/test/integrations/destinations/cordial/network.ts new file mode 100644 index 0000000000..9cc6f42408 --- /dev/null +++ b/test/integrations/destinations/cordial/network.ts @@ -0,0 +1,71 @@ +import { destination } from './common'; +export const networkCallsData = [ + { + httpReq: { + url: `${destination.Config.apiBaseUrl}/v2/contacts/email:johndoe@example.com`, + headers: { + Authorization: 'Basic dGVzdC1hcGkta2V5Og==', + }, + method: 'GET', + }, + httpRes: { + data: { + _id: '6690fe3655e334d6270287b5', + attributes: { + createdAt: '2024-07-12T09:58:14+0000', + address: { + city: 'San Miego', + }, + first_name: 'John', + last_name: 'Doe', + lastUpdateSource: 'api', + lastModified: '2024-07-12T13:00:49+0000', + cID: '6690fe3655e334d6270287b5', + }, + channels: { + email: { + address: 'johndoe@example.com', + subscribeStatus: 'subscribed', + subscribedAt: '2024-07-12T09:58:14+0000', + }, + }, + }, + status: 200, + statusText: 'Ok', + }, + }, + { + httpReq: { + url: `${destination.Config.apiBaseUrl}/v2/contacts/6690fe3655e334d6270287b5`, + headers: { + Authorization: 'Basic dGVzdC1hcGkta2V5Og==', + }, + method: 'GET', + }, + httpRes: { + data: { + _id: '6690fe3655e334d6270287b5', + attributes: { + createdAt: '2024-07-12T09:58:14+0000', + address: { + city: 'San Miego', + }, + first_name: 'John', + last_name: 'Doe', + lastUpdateSource: 'api', + lastModified: '2024-07-12T13:00:49+0000', + cID: '6690fe3655e334d6270287b5', + }, + channels: { + email: { + address: 'johndoe@example.com', + subscribeStatus: 'subscribed', + subscribedAt: '2024-07-12T09:58:14+0000', + }, + }, + }, + status: 200, + statusText: 'Ok', + }, + }, +]; diff --git a/test/integrations/destinations/cordial/processor/data.ts b/test/integrations/destinations/cordial/processor/data.ts new file mode 100644 index 0000000000..5c89c89a08 --- /dev/null +++ b/test/integrations/destinations/cordial/processor/data.ts @@ -0,0 +1,4 @@ +import { identify } from './identify'; +import { track } from './track'; +import { validation } from './validation'; +export const data = [...identify, ...track, ...validation]; diff --git a/test/integrations/destinations/cordial/processor/identify.ts b/test/integrations/destinations/cordial/processor/identify.ts new file mode 100644 index 0000000000..074852b199 --- /dev/null +++ b/test/integrations/destinations/cordial/processor/identify.ts @@ -0,0 +1,205 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { + destType, + destination, + traits, + headers, + endpoint, + updateContactEmailEndpoint, + updateContactIdEndpoint, + context, +} from '../common'; + +export const identify: ProcessorTestData[] = [ + { + id: 'cordial-identify-test-1', + name: destType, + description: 'Identify call to create contact', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + userId: 'userId123', + anonymousId: 'anonId123', + traits: { ...traits, email: 'abc@example.com' }, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + channels: { + email: { + address: 'abc@example.com', + subscribeStatus: 'subscribed', + }, + }, + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + address: { + city: 'New York', + country: 'USA', + pin_code: '123456', + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'cordial-identify-test-2', + name: destType, + description: 'Identify call to update contact using email', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + userId: 'userId123', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'PUT', + userId: '', + endpoint: updateContactEmailEndpoint, + headers, + JSON: { + channels: { + email: { + address: 'johndoe@example.com', + subscribeStatus: 'subscribed', + }, + }, + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + address: { + city: 'New York', + country: 'USA', + pin_code: '123456', + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'cordial-identify-test-3', + name: destType, + description: 'Identify call to update contact using contact id', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + userId: 'userId123', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + context, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'PUT', + userId: '', + endpoint: updateContactIdEndpoint, + headers, + JSON: { + channels: { + email: { + address: 'johndoe@example.com', + subscribeStatus: 'subscribed', + }, + }, + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + address: { + city: 'New York', + country: 'USA', + pin_code: '123456', + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/cordial/processor/track.ts b/test/integrations/destinations/cordial/processor/track.ts new file mode 100644 index 0000000000..3e7a560a52 --- /dev/null +++ b/test/integrations/destinations/cordial/processor/track.ts @@ -0,0 +1,125 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { + destType, + destination, + traits, + headers, + eventsEndpoint, + context, + properties, +} from '../common'; + +export const track: ProcessorTestData[] = [ + { + id: 'cordial-track-test-1', + name: destType, + description: 'Track event with exiting contact using email', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + userId: 'userId123', + anonymousId: 'anonId123', + event: 'test event', + properties, + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: eventsEndpoint, + headers, + JSON: { + a: 'test event', + email: 'johndoe@example.com', + ats: '2024-03-04T15:32:56.409Z', + properties, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'cordial-track-test-2', + name: destType, + description: 'Track event with existing contact using contact id', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + userId: 'userId123', + anonymousId: 'anonId123', + event: 'test event', + properties, + traits, + context, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: eventsEndpoint, + headers, + JSON: { + a: 'test event', + cID: '6690fe3655e334d6270287b5', + ats: '2024-03-04T15:32:56.409Z', + properties, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/cordial/processor/validation.ts b/test/integrations/destinations/cordial/processor/validation.ts new file mode 100644 index 0000000000..a764b79443 --- /dev/null +++ b/test/integrations/destinations/cordial/processor/validation.ts @@ -0,0 +1,50 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata } from '../../../testUtils'; +import { destType, destination, processorInstrumentationErrorStatTags } from '../common'; + +export const validation: ProcessorTestData[] = [ + { + id: 'cordial-validation-test-2', + name: destType, + description: 'Unsupported message type -> group', + scenario: 'Framework', + successCriteria: 'Instrumentation Error for Unsupported message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'group', + userId: 'userId123', + channel: 'mobile', + anonymousId: 'anon_123', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/cordial/router/data.ts b/test/integrations/destinations/cordial/router/data.ts new file mode 100644 index 0000000000..43d366d873 --- /dev/null +++ b/test/integrations/destinations/cordial/router/data.ts @@ -0,0 +1,134 @@ +import { generateMetadata } from '../../../testUtils'; +import { + destType, + destination, + traits, + properties, + headers, + eventsEndpoint, + updateContactEmailEndpoint, +} from '../common'; + +export const data = [ + { + id: 'cordial-router-test-1', + name: destType, + description: 'Basic Router Test to test multiple payloads', + scenario: 'Framework', + successCriteria: 'All events should be transformed successfully and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + type: 'identify', + userId: 'userId123', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'track', + userId: 'userId123', + anonymousId: 'anonId123', + event: 'test event', + properties, + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(2), + destination, + }, + ], + destType, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + body: { + XML: {}, + JSON_ARRAY: {}, + JSON: { + channels: { + email: { + address: 'johndoe@example.com', + subscribeStatus: 'subscribed', + }, + }, + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + address: { + city: 'New York', + country: 'USA', + pin_code: '123456', + }, + }, + FORM: {}, + }, + files: {}, + endpoint: updateContactEmailEndpoint, + headers, + version: '1', + params: {}, + type: 'REST', + method: 'PUT', + }, + metadata: [generateMetadata(1)], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: { + body: { + XML: {}, + JSON_ARRAY: {}, + JSON: { + a: 'test event', + email: 'johndoe@example.com', + ats: '2024-03-04T15:32:56.409Z', + properties, + }, + FORM: {}, + }, + files: {}, + endpoint: eventsEndpoint, + headers, + version: '1', + params: {}, + type: 'REST', + method: 'POST', + }, + metadata: [generateMetadata(2)], + batched: false, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + }, +]; From e357141d22e5296b6d1cda2e763ac24abfcb66e6 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 22 Jul 2024 11:10:05 +0530 Subject: [PATCH 06/28] feat: braze source event mapping (#3527) * feat: braze source event mapping * feat: delete unnecessary code * fix: removing the duplicate code from v0 route * fix: shortened the test cases --- src/v0/sources/braze/eventMapping.json | 1 - src/{v0 => v1}/sources/braze/ignore.json | 0 src/{v0 => v1}/sources/braze/mapping.json | 0 src/{v0 => v1}/sources/braze/transform.js | 29 +- test/__tests__/braze_source.test.js | 30 - test/__tests__/data/braze_source_input.json | 243 ---- test/__tests__/data/braze_source_output.json | 288 ---- test/integrations/sources/braze/common.ts | 30 + test/integrations/sources/braze/data.ts | 1252 ++++++++++++++++++ 9 files changed, 1297 insertions(+), 576 deletions(-) delete mode 100644 src/v0/sources/braze/eventMapping.json rename src/{v0 => v1}/sources/braze/ignore.json (100%) rename src/{v0 => v1}/sources/braze/mapping.json (100%) rename src/{v0 => v1}/sources/braze/transform.js (79%) delete mode 100644 test/__tests__/braze_source.test.js delete mode 100644 test/__tests__/data/braze_source_input.json delete mode 100644 test/__tests__/data/braze_source_output.json create mode 100644 test/integrations/sources/braze/common.ts create mode 100644 test/integrations/sources/braze/data.ts diff --git a/src/v0/sources/braze/eventMapping.json b/src/v0/sources/braze/eventMapping.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/src/v0/sources/braze/eventMapping.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/src/v0/sources/braze/ignore.json b/src/v1/sources/braze/ignore.json similarity index 100% rename from src/v0/sources/braze/ignore.json rename to src/v1/sources/braze/ignore.json diff --git a/src/v0/sources/braze/mapping.json b/src/v1/sources/braze/mapping.json similarity index 100% rename from src/v0/sources/braze/mapping.json rename to src/v1/sources/braze/mapping.json diff --git a/src/v0/sources/braze/transform.js b/src/v1/sources/braze/transform.js similarity index 79% rename from src/v0/sources/braze/transform.js rename to src/v1/sources/braze/transform.js index e3d7200023..771c5887b3 100644 --- a/src/v0/sources/braze/transform.js +++ b/src/v1/sources/braze/transform.js @@ -3,17 +3,15 @@ const get = require('get-value'); const path = require('path'); const fs = require('fs'); const { TransformationError } = require('@rudderstack/integrations-lib'); -const { formatTimeStamp, removeUndefinedAndNullValues } = require('../../util'); -const Message = require('../message'); +const { + formatTimeStamp, + removeUndefinedAndNullValues, + getHashFromArray, +} = require('../../../v0/util'); +const Message = require('../../../v0/sources/message'); // import mapping json using JSON.parse to preserve object key order const mapping = JSON.parse(fs.readFileSync(path.resolve(__dirname, './mapping.json'), 'utf-8')); - -// if we need to map braze event name to something else. blank as of now -const eventNameMap = JSON.parse( - fs.readFileSync(path.resolve(__dirname, './eventMapping.json'), 'utf-8'), -); - // ignored properties // to be deleted from the field `event.properties` as already mapped // using mapping.json @@ -21,7 +19,7 @@ const ignoredProperties = JSON.parse( fs.readFileSync(path.resolve(__dirname, './ignore.json'), 'utf-8'), ); -const processEvent = (event) => { +const processEvent = (event, eventMapping) => { const messageType = 'track'; if (event.event_type) { @@ -32,7 +30,7 @@ const processEvent = (event) => { message.setEventType(messageType); // set event name - const eventName = eventNameMap[eventType] || eventType; + const eventName = eventMapping[eventType] || eventType; message.setEventName(eventName); // map event properties based on mapping.json @@ -68,14 +66,17 @@ const processEvent = (event) => { throw new TransformationError('Unknown event type from Braze'); }; -const process = (events) => { +const process = (inputEvent) => { + const { event, source } = inputEvent; + const { customMapping } = source.Config; + const eventMapping = getHashFromArray(customMapping, 'from', 'to', false); const responses = []; // Ref: Custom Currents Connector Partner Dev Documentation.pdf - const eventList = Array.isArray(events) && events.length > 0 ? events[0].events : events.events; - eventList.forEach((event) => { + const eventList = Array.isArray(event) && event.length > 0 ? event[0].events : event.events; + eventList.forEach((singleEvent) => { try { - const resp = processEvent(event); + const resp = processEvent(singleEvent, eventMapping); if (resp) { responses.push(removeUndefinedAndNullValues(resp)); } diff --git a/test/__tests__/braze_source.test.js b/test/__tests__/braze_source.test.js deleted file mode 100644 index 009b650578..0000000000 --- a/test/__tests__/braze_source.test.js +++ /dev/null @@ -1,30 +0,0 @@ -const integration = "braze"; -const name = "Braze"; - -const fs = require("fs"); -const path = require("path"); - -const version = "v0"; - -const transformer = require(`../../src/${version}/sources/${integration}/transform`); - -const inputDataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_source_input.json`) -); -const outputDataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_source_output.json`) -); - -const inputData = JSON.parse(inputDataFile); -const expectedData = JSON.parse(outputDataFile); - -inputData.forEach((input, index) => { - it(`${name} Tests: payload: ${index}`, () => { - try { - const output = transformer.process(input); - expect(output).toEqual(expectedData[index]); - } catch (error) { - expect(error.message).toEqual(expectedData[index].message); - } - }); -}); diff --git a/test/__tests__/data/braze_source_input.json b/test/__tests__/data/braze_source_input.json deleted file mode 100644 index aef068229b..0000000000 --- a/test/__tests__/data/braze_source_input.json +++ /dev/null @@ -1,243 +0,0 @@ -[ - [ - { - "events": [ - { - "event_type": "users.messages.inappmessage.Click", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1607988752, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "device_id": "fedcba87-6543-210f-edc-ba9876543210", - "timezone": "America/Chicago" - }, - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "campaign_id": "11234567-89ab-cdef-0123-456789abcdef", - "campaign_name": "Test Campaign", - "message_variation_id": "c1234567-89ab-cdef-0123-456789abcdef", - "platform": "android", - "os_version": "Android (N)", - "device_model": "Nexus 5X", - "button_id": "0", - "send_id": "f123456789abcdef01234567" - } - }, - { - "event_type": "users.messages.pushnotification.Send", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1477502783, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "device_id": "fedcba87-6543-210f-edc-ba9876543210", - "timezone": "America/Chicago" - }, - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "platform": "ios", - "campaign_id": "11234567-89ab-cdef-0123-456789abcdef", - "campaign_name": "Test Campaign", - "message_variation_id": "c1234567-89ab-cdef-0123-456789abcdef", - "send_id": "f123456789abcdef01234567", - "dispatch_id": "01234567-89ab-cdef-0123-456789abcdef" - } - }, - { - "event_type": "users.messages.email.Open", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1477502783, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "timezone": "America/Chicago" - }, - "properties": { - "campaign_id": "11234567-89ab-cdef-0123-456789abcdef", - "campaign_name": "Test Campaign", - "dispatch_id": "12345qwert", - "message_variation_id": "c1234567-89ab-cdef-0123-456789abcdef", - "email_address": "test@test.com", - "send_id": "f123456789abcdef01234567", - "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36" - } - }, - { - "event_type": "users.messages.sms.Delivery", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1477502783, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "timezone": "America/Chicago" - }, - "properties": { - "campaign_id": "11234567-89ab-cdef-0123-456789abcdef", - "campaign_name": "Test Campaign", - "dispatch_id": "12345qwert", - "message_variation_id": "c1234567-89ab-cdef-0123-456789abcdef", - "to_phone_number": "+16462345678", - "subscription_group_id": "41234567-89ab-cdef-0123-456789abcdef", - "from_phone_number": "+12123470922" - } - }, - { - "event_type": "users.messages.inappmessage.Click", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1477502783, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "device_id": "fedcba87-6543-210f-edc-ba9876543210", - "timezone": "America/Chicago" - }, - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "canvas_id": "11234567-89ab-cdef-0123-456789abcdef", - "canvas_name": "My Cool Campaign", - "canvas_variation_id": "31234567-89ab-cdef-0123-456789abcdef", - "canvas_step_id": "41234567-89ab-cdef-0123-456789abcdef", - "platform": "android", - "os_version": "Android (N)", - "device_model": "Nexus 5X", - "button_id": "0", - "send_id": "f123456789abcdef01234567" - } - }, - { - "event_type": "users.messages.pushnotification.Send", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1477502783, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "device_id": "fedcba87-6543-210f-edc-ba9876543210", - "timezone": "America/Chicago" - }, - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "platform": "ios", - "canvas_id": "11234567-89ab-cdef-0123-456789abcdef", - "canvas_name": "My Cool Campaign", - "canvas_variation_id": "31234567-89ab-cdef-0123-456789abcdef", - "canvas_step_id": "41234567-89ab-cdef-0123-456789abcdef", - "send_id": "f123456789abcdef01234567", - "dispatch_id": "01234567-89ab-cdef-0123-456789abcdef" - } - }, - { - "event_type": "users.messages.email.Open", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1477502783, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "timezone": "America/Chicago" - }, - "properties": { - "canvas_id": "11234567-89ab-cdef-0123-456789abcdef", - "canvas_name": "My Cool Canvas", - "canvas_variation_id": "31234567-89ab-cdef-0123-456789abcdef", - "canvas_step_id": "41234567-89ab-cdef-0123-456789abcdef", - "dispatch_id": "12345qwert", - "email_address": "test@test.com", - "send_id": "f123456789abcdef01234567", - "user_agent": "Mozilla/5.0(Macintosh;IntelMacOSX10_13_5)AppleWebKit/537.36(KHTML,likeGecko)Chrome/67.0.3396.99Safari/537.36" - } - }, - { - "event_type": "users.messages.sms.Delivery", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1477502783, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "timezone": "America/Chicago" - }, - "properties": { - "canvas_id": "11234567-89ab-cdef-0123-456789abcdef", - "canvas_name": "MyCoolCanvas", - "canvas_variation_id": "31234567-89ab-cdef-0123-456789abcdef", - "canvas_step_id": "41234567-89ab-cdef-0123-456789abcdef", - "dispatch_id": "12345qwert", - "to_phone_number": "+16462345678", - "subscription_group_id": "41234567-89ab-cdef-0123-456789abcdef", - "from_phone_number": "+12123470922" - } - }, - { - "event_type": "users.behaviors.CustomEvent", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1477502783, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "device_id": "fedcba87-6543-210f-edc-ba9876543210", - "timezone": "America/Chicago" - }, - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "platform": "ios", - "os_version": "iOS10.3.1", - "device_model": "iPhone7Plus", - "name": "customeventname", - "ad_id": "01234567-89ab-cdef-0123-456789abcdef", - "ad_id_type": "roku_ad_id", - "ad_tracking_enabled": true, - "custom_properties": { - "stringpropertyname": "a", - "numberpropertyname": 1, - "listpropertyname": ["a", "b"] - } - } - }, - { - "event_type": "users.behaviors.Purchase", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1477502783, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "device_id": "fedcba87-6543-210f-edc-ba9876543210", - "timezone": "America/Chicago" - }, - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "platform": "ios", - "os_version": "iOS10.3.1", - "device_model": "iPhone7Plus", - "product_id": "1234", - "price": 12.34, - "currency": "AED", - "ad_id": "01234567-89ab-cdef-0123-456789abcdef", - "ad_id_type": "roku_ad_id", - "ad_tracking_enabled": true, - "purchase_properties": { - "stringpropertyname": "a", - "numberpropertyname": 1, - "listpropertyname": ["a", "b"] - } - } - }, - { - "event_type": "users.behaviors.app.SessionStart", - "id": "a1234567-89ab-cdef-0123-456789abcdef", - "time": 1477502783, - "user": { - "user_id": "0123456789abcdef01234567", - "external_user_id": "user_id", - "device_id": "fedcba87-6543-210f-edc-ba9876543210" - }, - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "platform": "ios", - "os_version": "iOS10.3.1", - "device_model": "iPhone7Plus", - "session_id": "b1234567-89ab-cdef-0123-456789abcdef" - } - } - ] - } - ] -] diff --git a/test/__tests__/data/braze_source_output.json b/test/__tests__/data/braze_source_output.json deleted file mode 100644 index b4e61885fe..0000000000 --- a/test/__tests__/data/braze_source_output.json +++ /dev/null @@ -1,288 +0,0 @@ -[ - [ - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "integration": { "name": "Braze" }, - "device": { - "id": "fedcba87-6543-210f-edc-ba9876543210", - "model": "Nexus 5X" - }, - "timezone": "America/Chicago", - "os": { "version": "Android (N)", "name": "android" } - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.messages.inappmessage.Click", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "campaign_id": "11234567-89ab-cdef-0123-456789abcdef", - "campaign_name": "Test Campaign", - "message_variation_id": "c1234567-89ab-cdef-0123-456789abcdef", - "button_id": "0", - "send_id": "f123456789abcdef01234567" - }, - "timestamp": "2020-12-14T23:32:32.000Z" - }, - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "integration": { "name": "Braze" }, - "device": { "id": "fedcba87-6543-210f-edc-ba9876543210" }, - "timezone": "America/Chicago", - "os": { "name": "ios" } - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.messages.pushnotification.Send", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "campaign_id": "11234567-89ab-cdef-0123-456789abcdef", - "campaign_name": "Test Campaign", - "message_variation_id": "c1234567-89ab-cdef-0123-456789abcdef", - "send_id": "f123456789abcdef01234567", - "dispatch_id": "01234567-89ab-cdef-0123-456789abcdef" - }, - "timestamp": "2016-10-26T17:26:23.000Z" - }, - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "integration": { "name": "Braze" }, - "timezone": "America/Chicago", - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36" - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.messages.email.Open", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "traits": { "email": "test@test.com" }, - "properties": { - "campaign_id": "11234567-89ab-cdef-0123-456789abcdef", - "campaign_name": "Test Campaign", - "dispatch_id": "12345qwert", - "message_variation_id": "c1234567-89ab-cdef-0123-456789abcdef", - "send_id": "f123456789abcdef01234567" - }, - "timestamp": "2016-10-26T17:26:23.000Z" - }, - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "timezone": "America/Chicago", - "integration": { "name": "Braze" } - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.messages.sms.Delivery", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "traits": { "phone": "+16462345678" }, - "properties": { - "campaign_id": "11234567-89ab-cdef-0123-456789abcdef", - "campaign_name": "Test Campaign", - "dispatch_id": "12345qwert", - "message_variation_id": "c1234567-89ab-cdef-0123-456789abcdef", - "subscription_group_id": "41234567-89ab-cdef-0123-456789abcdef", - "from_phone_number": "+12123470922" - }, - "timestamp": "2016-10-26T17:26:23.000Z" - }, - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "integration": { "name": "Braze" }, - "device": { - "id": "fedcba87-6543-210f-edc-ba9876543210", - "model": "Nexus 5X" - }, - "timezone": "America/Chicago", - "os": { "version": "Android (N)", "name": "android" } - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.messages.inappmessage.Click", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "canvas_id": "11234567-89ab-cdef-0123-456789abcdef", - "canvas_name": "My Cool Campaign", - "canvas_variation_id": "31234567-89ab-cdef-0123-456789abcdef", - "canvas_step_id": "41234567-89ab-cdef-0123-456789abcdef", - "button_id": "0", - "send_id": "f123456789abcdef01234567" - }, - "timestamp": "2016-10-26T17:26:23.000Z" - }, - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "integration": { "name": "Braze" }, - "device": { "id": "fedcba87-6543-210f-edc-ba9876543210" }, - "timezone": "America/Chicago", - "os": { "name": "ios" } - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.messages.pushnotification.Send", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "canvas_id": "11234567-89ab-cdef-0123-456789abcdef", - "canvas_name": "My Cool Campaign", - "canvas_variation_id": "31234567-89ab-cdef-0123-456789abcdef", - "canvas_step_id": "41234567-89ab-cdef-0123-456789abcdef", - "send_id": "f123456789abcdef01234567", - "dispatch_id": "01234567-89ab-cdef-0123-456789abcdef" - }, - "timestamp": "2016-10-26T17:26:23.000Z" - }, - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "integration": { "name": "Braze" }, - "timezone": "America/Chicago", - "userAgent": "Mozilla/5.0(Macintosh;IntelMacOSX10_13_5)AppleWebKit/537.36(KHTML,likeGecko)Chrome/67.0.3396.99Safari/537.36" - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.messages.email.Open", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "traits": { "email": "test@test.com" }, - "properties": { - "canvas_id": "11234567-89ab-cdef-0123-456789abcdef", - "canvas_name": "My Cool Canvas", - "canvas_variation_id": "31234567-89ab-cdef-0123-456789abcdef", - "canvas_step_id": "41234567-89ab-cdef-0123-456789abcdef", - "dispatch_id": "12345qwert", - "send_id": "f123456789abcdef01234567" - }, - "timestamp": "2016-10-26T17:26:23.000Z" - }, - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "timezone": "America/Chicago", - "integration": { "name": "Braze" } - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.messages.sms.Delivery", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "traits": { "phone": "+16462345678" }, - "properties": { - "canvas_id": "11234567-89ab-cdef-0123-456789abcdef", - "canvas_name": "MyCoolCanvas", - "canvas_variation_id": "31234567-89ab-cdef-0123-456789abcdef", - "canvas_step_id": "41234567-89ab-cdef-0123-456789abcdef", - "dispatch_id": "12345qwert", - "subscription_group_id": "41234567-89ab-cdef-0123-456789abcdef", - "from_phone_number": "+12123470922" - }, - "timestamp": "2016-10-26T17:26:23.000Z" - }, - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "integration": { "name": "Braze" }, - "device": { - "id": "fedcba87-6543-210f-edc-ba9876543210", - "model": "iPhone7Plus", - "advertisingId": "01234567-89ab-cdef-0123-456789abcdef", - "adTrackingEnabled": true - }, - "timezone": "America/Chicago", - "os": { "version": "iOS10.3.1", "name": "ios" } - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.behaviors.CustomEvent", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "name": "customeventname", - "ad_id_type": "roku_ad_id", - "custom_properties": { - "stringpropertyname": "a", - "numberpropertyname": 1, - "listpropertyname": ["a", "b"] - } - }, - "timestamp": "2016-10-26T17:26:23.000Z" - }, - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "integration": { "name": "Braze" }, - "device": { - "id": "fedcba87-6543-210f-edc-ba9876543210", - "model": "iPhone7Plus", - "advertisingId": "01234567-89ab-cdef-0123-456789abcdef", - "adTrackingEnabled": true - }, - "timezone": "America/Chicago", - "os": { "version": "iOS10.3.1", "name": "ios" } - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.behaviors.Purchase", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "product_id": "1234", - "price": 12.34, - "currency": "AED", - "ad_id_type": "roku_ad_id", - "purchase_properties": { - "stringpropertyname": "a", - "numberpropertyname": 1, - "listpropertyname": ["a", "b"] - } - }, - "timestamp": "2016-10-26T17:26:23.000Z" - }, - { - "context": { - "library": { "name": "unknown", "version": "unknown" }, - "integration": { "name": "Braze" }, - "device": { - "id": "fedcba87-6543-210f-edc-ba9876543210", - "model": "iPhone7Plus" - }, - "os": { "version": "iOS10.3.1", "name": "ios" } - }, - "integrations": { "Braze": false }, - "type": "track", - "event": "users.behaviors.app.SessionStart", - "messageId": "a1234567-89ab-cdef-0123-456789abcdef", - "anonymousId": "0123456789abcdef01234567", - "userId": "user_id", - "properties": { - "app_id": "01234567-89ab-cdef-0123-456789abcdef", - "session_id": "b1234567-89ab-cdef-0123-456789abcdef" - }, - "timestamp": "2016-10-26T17:26:23.000Z" - } - ] -] diff --git a/test/integrations/sources/braze/common.ts b/test/integrations/sources/braze/common.ts new file mode 100644 index 0000000000..a8271e3e9c --- /dev/null +++ b/test/integrations/sources/braze/common.ts @@ -0,0 +1,30 @@ +export const commonSourceDefinition = { + ID: '1lh9senY3vrBg4JQXswWzyYBTOO', + Name: 'Braze', + Category: 'webhook', + Type: 'cloud', +}; + +export const DgSourceTrackingPlanConfig = { + sourceId: '', + version: 0, + config: null, + mergedConfig: null, + deleted: false, + trackingPlan: { + id: '', + version: 0, + }, +}; + +export const commonSourceConfigProperties = { + Enabled: true, + WorkspaceID: '2hSS1hZ8kuCpUZAAYsQucAFdob9', + Destinations: null, + WriteKey: '2hgvYykpvMaE5Eg47Au8RWC9Yza', + DgSourceTrackingPlanConfig: DgSourceTrackingPlanConfig, + Transient: false, + GeoEnrichment: { + Enabled: false, + }, +}; diff --git a/test/integrations/sources/braze/data.ts b/test/integrations/sources/braze/data.ts new file mode 100644 index 0000000000..e140bbc030 --- /dev/null +++ b/test/integrations/sources/braze/data.ts @@ -0,0 +1,1252 @@ +import { commonSourceConfigProperties, commonSourceDefinition } from './common'; + +export const data = [ + { + name: 'braze', + description: 'event mapping done in UI', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.messages.inappmessage.Click', + properties: { + device_model: 'samsung', + }, + user: { + user_id: 'user_id', + external_user_id: 'externalUserId', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'users.messages.inappmessage.Click', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + anonymousId: 'user_id', + context: { + device: { + model: 'samsung', + }, + integration: { + name: 'Braze', + }, + library: { + name: 'unknown', + version: 'unknown', + }, + }, + event: 'In-App Message Clicked', + integrations: { + Braze: false, + }, + type: 'track', + userId: 'externalUserId', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'The event is not mapped in the UI', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.messages.inappmessage.Click', + properties: { + device_model: 'samsung', + }, + user: { + user_id: 'user_id', + external_user_id: 'externalUserId', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + anonymousId: 'user_id', + context: { + device: { + model: 'samsung', + }, + integration: { + name: 'Braze', + }, + library: { + name: 'unknown', + version: 'unknown', + }, + }, + event: 'users.messages.inappmessage.Click', + integrations: { + Braze: false, + }, + type: 'track', + userId: 'externalUserId', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.messages.inappmessage.Click event', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.messages.inappmessage.Click', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1607988752, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + device_id: 'fedcba87-6543-210f-edc-ba9876543210', + timezone: 'America/Chicago', + }, + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + campaign_id: '11234567-89ab-cdef-0123-456789abcdef', + campaign_name: 'Test Campaign', + message_variation_id: 'c1234567-89ab-cdef-0123-456789abcdef', + platform: 'android', + os_version: 'Android (N)', + device_model: 'Nexus 5X', + button_id: '0', + send_id: 'f123456789abcdef01234567', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + integration: { name: 'Braze' }, + device: { + id: 'fedcba87-6543-210f-edc-ba9876543210', + model: 'Nexus 5X', + }, + timezone: 'America/Chicago', + os: { version: 'Android (N)', name: 'android' }, + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.messages.inappmessage.Click', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + campaign_id: '11234567-89ab-cdef-0123-456789abcdef', + campaign_name: 'Test Campaign', + message_variation_id: 'c1234567-89ab-cdef-0123-456789abcdef', + button_id: '0', + send_id: 'f123456789abcdef01234567', + }, + timestamp: '2020-12-14T23:32:32.000Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.messages.pushnotification.Send event', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.messages.pushnotification.Send', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1477502783, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + device_id: 'fedcba87-6543-210f-edc-ba9876543210', + timezone: 'America/Chicago', + }, + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + platform: 'ios', + campaign_id: '11234567-89ab-cdef-0123-456789abcdef', + campaign_name: 'Test Campaign', + message_variation_id: 'c1234567-89ab-cdef-0123-456789abcdef', + send_id: 'f123456789abcdef01234567', + dispatch_id: '01234567-89ab-cdef-0123-456789abcdef', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + integration: { name: 'Braze' }, + device: { id: 'fedcba87-6543-210f-edc-ba9876543210' }, + timezone: 'America/Chicago', + os: { name: 'ios' }, + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.messages.pushnotification.Send', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + campaign_id: '11234567-89ab-cdef-0123-456789abcdef', + campaign_name: 'Test Campaign', + message_variation_id: 'c1234567-89ab-cdef-0123-456789abcdef', + send_id: 'f123456789abcdef01234567', + dispatch_id: '01234567-89ab-cdef-0123-456789abcdef', + }, + timestamp: '2016-10-26T17:26:23.000Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.messages.email.Open event', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.messages.email.Open', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1477502783, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + timezone: 'America/Chicago', + }, + properties: { + campaign_id: '11234567-89ab-cdef-0123-456789abcdef', + campaign_name: 'Test Campaign', + dispatch_id: '12345qwert', + message_variation_id: 'c1234567-89ab-cdef-0123-456789abcdef', + email_address: 'test@test.com', + send_id: 'f123456789abcdef01234567', + user_agent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + integration: { name: 'Braze' }, + timezone: 'America/Chicago', + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36', + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.messages.email.Open', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + traits: { email: 'test@test.com' }, + properties: { + campaign_id: '11234567-89ab-cdef-0123-456789abcdef', + campaign_name: 'Test Campaign', + dispatch_id: '12345qwert', + message_variation_id: 'c1234567-89ab-cdef-0123-456789abcdef', + send_id: 'f123456789abcdef01234567', + }, + timestamp: '2016-10-26T17:26:23.000Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.messages.sms.Delivery send', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.messages.sms.Delivery', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1477502783, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + timezone: 'America/Chicago', + }, + properties: { + campaign_id: '11234567-89ab-cdef-0123-456789abcdef', + campaign_name: 'Test Campaign', + dispatch_id: '12345qwert', + message_variation_id: 'c1234567-89ab-cdef-0123-456789abcdef', + to_phone_number: '+16462345678', + subscription_group_id: '41234567-89ab-cdef-0123-456789abcdef', + from_phone_number: '+12123470922', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + timezone: 'America/Chicago', + integration: { name: 'Braze' }, + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.messages.sms.Delivery', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + traits: { phone: '+16462345678' }, + properties: { + campaign_id: '11234567-89ab-cdef-0123-456789abcdef', + campaign_name: 'Test Campaign', + dispatch_id: '12345qwert', + message_variation_id: 'c1234567-89ab-cdef-0123-456789abcdef', + subscription_group_id: '41234567-89ab-cdef-0123-456789abcdef', + from_phone_number: '+12123470922', + }, + timestamp: '2016-10-26T17:26:23.000Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.messages.inappmessage.Click event', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.messages.inappmessage.Click', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1477502783, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + device_id: 'fedcba87-6543-210f-edc-ba9876543210', + timezone: 'America/Chicago', + }, + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + canvas_id: '11234567-89ab-cdef-0123-456789abcdef', + canvas_name: 'My Cool Campaign', + canvas_variation_id: '31234567-89ab-cdef-0123-456789abcdef', + canvas_step_id: '41234567-89ab-cdef-0123-456789abcdef', + platform: 'android', + os_version: 'Android (N)', + device_model: 'Nexus 5X', + button_id: '0', + send_id: 'f123456789abcdef01234567', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + integration: { name: 'Braze' }, + device: { + id: 'fedcba87-6543-210f-edc-ba9876543210', + model: 'Nexus 5X', + }, + timezone: 'America/Chicago', + os: { version: 'Android (N)', name: 'android' }, + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.messages.inappmessage.Click', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + canvas_id: '11234567-89ab-cdef-0123-456789abcdef', + canvas_name: 'My Cool Campaign', + canvas_variation_id: '31234567-89ab-cdef-0123-456789abcdef', + canvas_step_id: '41234567-89ab-cdef-0123-456789abcdef', + button_id: '0', + send_id: 'f123456789abcdef01234567', + }, + timestamp: '2016-10-26T17:26:23.000Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.messages.pushnotification.Send event', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.messages.pushnotification.Send', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1477502783, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + device_id: 'fedcba87-6543-210f-edc-ba9876543210', + timezone: 'America/Chicago', + }, + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + platform: 'ios', + canvas_id: '11234567-89ab-cdef-0123-456789abcdef', + canvas_name: 'My Cool Campaign', + canvas_variation_id: '31234567-89ab-cdef-0123-456789abcdef', + canvas_step_id: '41234567-89ab-cdef-0123-456789abcdef', + send_id: 'f123456789abcdef01234567', + dispatch_id: '01234567-89ab-cdef-0123-456789abcdef', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + integration: { name: 'Braze' }, + device: { id: 'fedcba87-6543-210f-edc-ba9876543210' }, + timezone: 'America/Chicago', + os: { name: 'ios' }, + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.messages.pushnotification.Send', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + canvas_id: '11234567-89ab-cdef-0123-456789abcdef', + canvas_name: 'My Cool Campaign', + canvas_variation_id: '31234567-89ab-cdef-0123-456789abcdef', + canvas_step_id: '41234567-89ab-cdef-0123-456789abcdef', + send_id: 'f123456789abcdef01234567', + dispatch_id: '01234567-89ab-cdef-0123-456789abcdef', + }, + timestamp: '2016-10-26T17:26:23.000Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.messages.email.Open event', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.messages.email.Open', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1477502783, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + timezone: 'America/Chicago', + }, + properties: { + canvas_id: '11234567-89ab-cdef-0123-456789abcdef', + canvas_name: 'My Cool Canvas', + canvas_variation_id: '31234567-89ab-cdef-0123-456789abcdef', + canvas_step_id: '41234567-89ab-cdef-0123-456789abcdef', + dispatch_id: '12345qwert', + email_address: 'test@test.com', + send_id: 'f123456789abcdef01234567', + user_agent: + 'Mozilla/5.0(Macintosh;IntelMacOSX10_13_5)AppleWebKit/537.36(KHTML,likeGecko)Chrome/67.0.3396.99Safari/537.36', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + integration: { name: 'Braze' }, + timezone: 'America/Chicago', + userAgent: + 'Mozilla/5.0(Macintosh;IntelMacOSX10_13_5)AppleWebKit/537.36(KHTML,likeGecko)Chrome/67.0.3396.99Safari/537.36', + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.messages.email.Open', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + traits: { email: 'test@test.com' }, + properties: { + canvas_id: '11234567-89ab-cdef-0123-456789abcdef', + canvas_name: 'My Cool Canvas', + canvas_variation_id: '31234567-89ab-cdef-0123-456789abcdef', + canvas_step_id: '41234567-89ab-cdef-0123-456789abcdef', + dispatch_id: '12345qwert', + send_id: 'f123456789abcdef01234567', + }, + timestamp: '2016-10-26T17:26:23.000Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.messages.sms.Delivery event', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.messages.sms.Delivery', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1477502783, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + timezone: 'America/Chicago', + }, + properties: { + canvas_id: '11234567-89ab-cdef-0123-456789abcdef', + canvas_name: 'MyCoolCanvas', + canvas_variation_id: '31234567-89ab-cdef-0123-456789abcdef', + canvas_step_id: '41234567-89ab-cdef-0123-456789abcdef', + dispatch_id: '12345qwert', + to_phone_number: '+16462345678', + subscription_group_id: '41234567-89ab-cdef-0123-456789abcdef', + from_phone_number: '+12123470922', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + timezone: 'America/Chicago', + integration: { name: 'Braze' }, + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.messages.sms.Delivery', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + traits: { phone: '+16462345678' }, + properties: { + canvas_id: '11234567-89ab-cdef-0123-456789abcdef', + canvas_name: 'MyCoolCanvas', + canvas_variation_id: '31234567-89ab-cdef-0123-456789abcdef', + canvas_step_id: '41234567-89ab-cdef-0123-456789abcdef', + dispatch_id: '12345qwert', + subscription_group_id: '41234567-89ab-cdef-0123-456789abcdef', + from_phone_number: '+12123470922', + }, + timestamp: '2016-10-26T17:26:23.000Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.behaviors.CustomEvent any custom event', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.behaviors.CustomEvent', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1477502783, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + device_id: 'fedcba87-6543-210f-edc-ba9876543210', + timezone: 'America/Chicago', + }, + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + platform: 'ios', + os_version: 'iOS10.3.1', + device_model: 'iPhone7Plus', + name: 'customeventname', + ad_id: '01234567-89ab-cdef-0123-456789abcdef', + ad_id_type: 'roku_ad_id', + ad_tracking_enabled: true, + custom_properties: { + stringpropertyname: 'a', + numberpropertyname: 1, + listpropertyname: ['a', 'b'], + }, + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + integration: { name: 'Braze' }, + device: { + id: 'fedcba87-6543-210f-edc-ba9876543210', + model: 'iPhone7Plus', + advertisingId: '01234567-89ab-cdef-0123-456789abcdef', + adTrackingEnabled: true, + }, + timezone: 'America/Chicago', + os: { version: 'iOS10.3.1', name: 'ios' }, + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.behaviors.CustomEvent', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + name: 'customeventname', + ad_id_type: 'roku_ad_id', + custom_properties: { + stringpropertyname: 'a', + numberpropertyname: 1, + listpropertyname: ['a', 'b'], + }, + }, + timestamp: '2016-10-26T17:26:23.000Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.behaviors.Purchase event', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.behaviors.Purchase', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1477502783, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + device_id: 'fedcba87-6543-210f-edc-ba9876543210', + timezone: 'America/Chicago', + }, + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + platform: 'ios', + os_version: 'iOS10.3.1', + device_model: 'iPhone7Plus', + product_id: '1234', + price: 12.34, + currency: 'AED', + ad_id: '01234567-89ab-cdef-0123-456789abcdef', + ad_id_type: 'roku_ad_id', + ad_tracking_enabled: true, + purchase_properties: { + stringpropertyname: 'a', + numberpropertyname: 1, + listpropertyname: ['a', 'b'], + }, + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + integration: { name: 'Braze' }, + device: { + id: 'fedcba87-6543-210f-edc-ba9876543210', + model: 'iPhone7Plus', + advertisingId: '01234567-89ab-cdef-0123-456789abcdef', + adTrackingEnabled: true, + }, + timezone: 'America/Chicago', + os: { version: 'iOS10.3.1', name: 'ios' }, + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.behaviors.Purchase', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + product_id: '1234', + price: 12.34, + currency: 'AED', + ad_id_type: 'roku_ad_id', + purchase_properties: { + stringpropertyname: 'a', + numberpropertyname: 1, + listpropertyname: ['a', 'b'], + }, + }, + timestamp: '2016-10-26T17:26:23.000Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'braze', + description: 'users.behaviors.app.SessionStart event', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + events: [ + { + event_type: 'users.behaviors.app.SessionStart', + id: 'a1234567-89ab-cdef-0123-456789abcdef', + time: 1477502783, + user: { + user_id: '0123456789abcdef01234567', + external_user_id: 'user_id', + device_id: 'fedcba87-6543-210f-edc-ba9876543210', + }, + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + platform: 'ios', + os_version: 'iOS10.3.1', + device_model: 'iPhone7Plus', + session_id: 'b1234567-89ab-cdef-0123-456789abcdef', + }, + }, + ], + }, + source: { + ID: '2hgvYyU5TYaFvVzBge6tF2UKoeG', + OriginalID: '', + Name: 'Braze source', + SourceDefinition: commonSourceDefinition, + Config: { + customMapping: [ + { + from: 'randomEvent', + to: 'In-App Message Clicked', + }, + ], + }, + ...commonSourceConfigProperties, + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { name: 'unknown', version: 'unknown' }, + integration: { name: 'Braze' }, + device: { + id: 'fedcba87-6543-210f-edc-ba9876543210', + model: 'iPhone7Plus', + }, + os: { version: 'iOS10.3.1', name: 'ios' }, + }, + integrations: { Braze: false }, + type: 'track', + event: 'users.behaviors.app.SessionStart', + messageId: 'a1234567-89ab-cdef-0123-456789abcdef', + anonymousId: '0123456789abcdef01234567', + userId: 'user_id', + properties: { + app_id: '01234567-89ab-cdef-0123-456789abcdef', + session_id: 'b1234567-89ab-cdef-0123-456789abcdef', + }, + timestamp: '2016-10-26T17:26:23.000Z', + }, + ], + }, + }, + ], + }, + }, + }, +]; From 20aa7f35e13ad89e8a43fbbb743df73b0c103975 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:14:44 +0530 Subject: [PATCH 07/28] feat: onboarding new destination zoho (#3555) * feat: initial commit * feat: adding unit test for utils * feat: adding router tests * feat: editing batching logic * feat: adding data delivery test cases * fix: fixing the endpoint bugs * feat: zoho record deletion feature (#3566) * chore: binding issue code * fix: searchRecordId function * fix: adding record deletion implementation * fix: adding record deletion test cases --------- Co-authored-by: Dilip Kola * chore: adding debug logs * fix: code refactor * fix: shortening the test cases * fix: error message edit * chore: missing comma in features file * fix: adding validation for inconsistent module choice * fix: extra fields and logs removed --------- Co-authored-by: Dilip Kola --- src/cdk/v2/destinations/zoho/config.js | 61 ++ src/cdk/v2/destinations/zoho/rtWorkflow.yaml | 38 + .../v2/destinations/zoho/transformRecord.js | 338 ++++++++ src/cdk/v2/destinations/zoho/utils.js | 168 ++++ src/cdk/v2/destinations/zoho/utils.test.js | 245 ++++++ src/features.json | 1 + src/v1/destinations/zoho/networkHandler.js | 141 ++++ test/integrations/component.test.ts | 4 +- test/integrations/destinations/zoho/common.ts | 334 ++++++++ .../zoho/dataDelivery/business.ts | 401 +++++++++ .../destinations/zoho/dataDelivery/data.ts | 3 + test/integrations/destinations/zoho/mocks.ts | 5 + .../integrations/destinations/zoho/network.ts | 421 ++++++++++ .../destinations/zoho/router/data.ts | 4 + .../destinations/zoho/router/deletion.ts | 249 ++++++ .../destinations/zoho/router/upsert.ts | 771 ++++++++++++++++++ 16 files changed, 3183 insertions(+), 1 deletion(-) create mode 100644 src/cdk/v2/destinations/zoho/config.js create mode 100644 src/cdk/v2/destinations/zoho/rtWorkflow.yaml create mode 100644 src/cdk/v2/destinations/zoho/transformRecord.js create mode 100644 src/cdk/v2/destinations/zoho/utils.js create mode 100644 src/cdk/v2/destinations/zoho/utils.test.js create mode 100644 src/v1/destinations/zoho/networkHandler.js create mode 100644 test/integrations/destinations/zoho/common.ts create mode 100644 test/integrations/destinations/zoho/dataDelivery/business.ts create mode 100644 test/integrations/destinations/zoho/dataDelivery/data.ts create mode 100644 test/integrations/destinations/zoho/mocks.ts create mode 100644 test/integrations/destinations/zoho/network.ts create mode 100644 test/integrations/destinations/zoho/router/data.ts create mode 100644 test/integrations/destinations/zoho/router/deletion.ts create mode 100644 test/integrations/destinations/zoho/router/upsert.ts diff --git a/src/cdk/v2/destinations/zoho/config.js b/src/cdk/v2/destinations/zoho/config.js new file mode 100644 index 0000000000..d942d9e369 --- /dev/null +++ b/src/cdk/v2/destinations/zoho/config.js @@ -0,0 +1,61 @@ +// https://www.zoho.com/crm/developer/docs/api/v6/access-refresh.html +const DATA_CENTRE_BASE_ENDPOINTS_MAP = { + US: 'https://www.zohoapis.com', + AU: 'https://www.zohoapis.com.au', + EU: 'https://www.zohoapis.eu', + IN: 'https://www.zohoapis.in', + CN: 'https://www.zohoapis.com.cn', + JP: 'https://www.zohoapis.jp', + CA: 'https://www.zohoapiscloud.ca', +}; + +const getBaseEndpoint = (dataServer) => DATA_CENTRE_BASE_ENDPOINTS_MAP[dataServer]; +const COMMON_RECORD_ENDPOINT = (dataCenter = 'US') => + `${getBaseEndpoint(dataCenter)}/crm/v6/moduleType`; + +// ref: https://www.zoho.com/crm/developer/docs/api/v6/insert-records.html#:~:text=%2DX%20POST-,System%2Ddefined%20mandatory%20fields%20for%20each%20module,-While%20inserting%20records +const MODULE_MANDATORY_FIELD_CONFIG = { + Leads: ['Last_Name'], + Contacts: ['Last_Name'], + Accounts: ['Account_Name'], + Deals: ['Deal_Name', 'Stage', 'Pipeline'], + Tasks: ['Subject'], + Calls: ['Subject', 'Call_Type', 'Call_Start_Time', 'Call_Duration'], + Events: ['Event_Title', 'Start_DateTime', 'Remind_At', 'End_DateTime'], + Products: ['Product_Name'], + Quotes: ['Subject', 'Quoted_Items'], + Invoices: ['Subject', 'Invoiced_Items'], + Campaigns: ['Campaign_Name'], + Vendors: ['Vendor_Name'], + 'Price Books': ['Price_Book_Name', 'Pricing_Details'], + Cases: ['Case_Origin', 'Status', 'Subject'], + Solutions: ['Solution_Title'], + 'Purchase Orders': ['Subject', 'Vendor_Name', 'Purchased_Items'], + 'Sales Orders': ['Subject', 'Ordered_Items'], +}; + +const MODULE_WISE_DUPLICATE_CHECK_FIELD = { + Leads: ['Email'], + Accounts: ['Account_Name'], + Contacts: ['Email'], + Deals: ['Deal_Name'], + Campaigns: ['Campaign_Name'], + Cases: ['Subject'], + Solutions: ['Solution_Title'], + Products: ['Product_Name'], + Vendors: ['Vendor_Name'], + PriceBooks: ['Price_Book_Name'], + Quotes: ['Subject'], + SalesOrders: ['Subject'], + PurchaseOrders: ['Subject'], + Invoices: ['Subject'], + CustomModules: ['Name'], +}; + +module.exports = { + MAX_BATCH_SIZE: 100, + DATA_CENTRE_BASE_ENDPOINTS_MAP, + COMMON_RECORD_ENDPOINT, + MODULE_MANDATORY_FIELD_CONFIG, + MODULE_WISE_DUPLICATE_CHECK_FIELD, +}; diff --git a/src/cdk/v2/destinations/zoho/rtWorkflow.yaml b/src/cdk/v2/destinations/zoho/rtWorkflow.yaml new file mode 100644 index 0000000000..b50b9502e3 --- /dev/null +++ b/src/cdk/v2/destinations/zoho/rtWorkflow.yaml @@ -0,0 +1,38 @@ +bindings: + - name: EventType + path: ../../../../constants + - name: processRecordInputs + path: ./transformRecord + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - name: InstrumentationError + path: '@rudderstack/integrations-lib' + +steps: + - name: validateConfig + template: | + const config = ^[0].destination.Config + $.assertConfig(config.region, "Datacentre Region is not present. Aborting") + + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: processRecordEvents + template: | + await $.processRecordInputs(^.{.message.type === $.EventType.RECORD}[], ^[0].destination) + + - name: failOtherEvents + template: | + const otherEvents = ^.{.message.type !== $.EventType.RECORD}[] + let failedEvents = otherEvents.map( + function(event) { + const error = new $.InstrumentationError("Event type " + event.message.type + " is not supported"); + $.handleRtTfSingleEventError(event, error, {}) + } + ) + failedEvents ?? [] + + - name: finalPayload + template: | + [...$.outputs.processRecordEvents, ...$.outputs.failOtherEvents] diff --git a/src/cdk/v2/destinations/zoho/transformRecord.js b/src/cdk/v2/destinations/zoho/transformRecord.js new file mode 100644 index 0000000000..8f4586e46b --- /dev/null +++ b/src/cdk/v2/destinations/zoho/transformRecord.js @@ -0,0 +1,338 @@ +const { + InstrumentationError, + getHashFromArray, + ConfigurationError, + RetryableError, +} = require('@rudderstack/integrations-lib'); +const { BatchUtils } = require('@rudderstack/workflow-engine'); +const { + defaultPostRequestConfig, + defaultRequestConfig, + getSuccessRespEvents, + removeUndefinedAndNullValues, + handleRtTfSingleEventError, + isEmptyObject, + defaultDeleteRequestConfig, +} = require('../../../../v0/util'); +const zohoConfig = require('./config'); +const { + deduceModuleInfo, + validatePresenceOfMandatoryProperties, + formatMultiSelectFields, + handleDuplicateCheck, + searchRecordId, + calculateTrigger, + validateConfigurationIssue, +} = require('./utils'); +const { REFRESH_TOKEN } = require('../../../../adapters/networkhandler/authConstants'); + +// Main response builder function +const responseBuilder = ( + items, + config, + identifierType, + operationModuleType, + commonEndPoint, + action, + metadata, +) => { + const { trigger, addDefaultDuplicateCheck, multiSelectFieldLevelDecision } = config; + + const response = defaultRequestConfig(); + response.headers = { + Authorization: `Zoho-oauthtoken ${metadata[0].secret.accessToken}`, + }; + + if (action === 'insert' || action === 'update') { + const payload = { + duplicate_check_fields: handleDuplicateCheck( + addDefaultDuplicateCheck, + identifierType, + operationModuleType, + ), + data: items, + $append_values: getHashFromArray(multiSelectFieldLevelDecision, 'from', 'to', false), + trigger: calculateTrigger(trigger), + }; + response.method = defaultPostRequestConfig.requestMethod; + response.body.JSON = removeUndefinedAndNullValues(payload); + response.endpoint = `${commonEndPoint}/upsert`; + } else { + response.endpoint = `${commonEndPoint}?ids=${items.join(',')}&wf_trigger=${trigger !== 'None'}`; + response.method = defaultDeleteRequestConfig.requestMethod; + } + + return response; +}; +const batchResponseBuilder = ( + transformedResponseToBeBatched, + config, + identifierType, + operationModuleType, + upsertEndPoint, + action, +) => { + const upsertResponseArray = []; + const deletionResponseArray = []; + const { upsertData, deletionData, upsertSuccessMetadata, deletionSuccessMetadata } = + transformedResponseToBeBatched; + + const upsertDataChunks = BatchUtils.chunkArrayBySizeAndLength(upsertData, { + maxItems: zohoConfig.MAX_BATCH_SIZE, + }); + + const deletionDataChunks = BatchUtils.chunkArrayBySizeAndLength(deletionData, { + maxItems: zohoConfig.MAX_BATCH_SIZE, + }); + + const upsertmetadataChunks = BatchUtils.chunkArrayBySizeAndLength(upsertSuccessMetadata, { + maxItems: zohoConfig.MAX_BATCH_SIZE, + }); + + const deletionmetadataChunks = BatchUtils.chunkArrayBySizeAndLength(deletionSuccessMetadata, { + maxItems: zohoConfig.MAX_BATCH_SIZE, + }); + + upsertDataChunks.items.forEach((chunk) => { + upsertResponseArray.push( + responseBuilder( + chunk, + config, + identifierType, + operationModuleType, + upsertEndPoint, + action, + upsertmetadataChunks.items[0], + ), + ); + }); + + deletionDataChunks.items.forEach((chunk) => { + deletionResponseArray.push( + responseBuilder( + chunk, + config, + identifierType, + operationModuleType, + upsertEndPoint, + action, + deletionmetadataChunks.items[0], + ), + ); + }); + + return { + upsertResponseArray, + upsertmetadataChunks, + deletionResponseArray, + deletionmetadataChunks, + }; +}; + +/** + * Handles the upsert operation for a specific module type by validating mandatory properties, + * processing the input fields, and updating the response accordingly. + * + * @param {Object} input - The input data for the upsert operation. + * @param {Object} fields - The fields to be upserted. + * @param {string} operationModuleType - The type of module operation being performed. + * @param {Object} Config - The configuration object. + * @param {Object} transformedResponseToBeBatched - The response object to be batched. + * @param {Array} errorResponseList - The list to store error responses. + * @returns {Promise} - A promise that resolves once the upsert operation is handled. + */ +const handleUpsert = async ( + input, + fields, + operationModuleType, + Config, + transformedResponseToBeBatched, + errorResponseList, +) => { + const eventErroneous = validatePresenceOfMandatoryProperties(operationModuleType, fields); + + if (eventErroneous?.status) { + const error = new ConfigurationError( + `${operationModuleType} object must have the ${eventErroneous.missingField.join('", "')} property(ies).`, + ); + errorResponseList.push(handleRtTfSingleEventError(input, error, {})); + } else { + const formattedFields = formatMultiSelectFields(Config, fields); + transformedResponseToBeBatched.upsertSuccessMetadata.push(input.metadata); + transformedResponseToBeBatched.upsertData.push(formattedFields); + } +}; + +/** + * Handles search errors in Zoho record search. + * If the search response message code is 'INVALID_TOKEN', returns a RetryableError with a specific message and status code. + * Otherwise, returns a ConfigurationError with a message indicating failure to fetch Zoho ID for a record. + * + * @param {Object} searchResponse - The response object from the search operation. + * @returns {RetryableError|ConfigurationError} - The error object based on the search response. + */ +const handleSearchError = (searchResponse) => { + if (searchResponse.message.code === 'INVALID_TOKEN') { + return new RetryableError( + `[Zoho]:: ${JSON.stringify(searchResponse.message)} during zoho record search`, + 500, + searchResponse.message, + REFRESH_TOKEN, + ); + } + return new ConfigurationError( + `failed to fetch zoho id for record for ${JSON.stringify(searchResponse.message)}`, + ); +}; + +/** + * Asynchronously handles the deletion operation based on the search response. + * + * @param {Object} input - The input object containing metadata and other details. + * @param {Array} fields - The fields to be used for searching the record. + * @param {Object} Config - The configuration object. + * @param {Object} transformedResponseToBeBatched - The object to store transformed response data to be batched. + * @param {Array} errorResponseList - The list to store error responses. + */ +const handleDeletion = async ( + input, + fields, + Config, + transformedResponseToBeBatched, + errorResponseList, +) => { + const searchResponse = await searchRecordId(fields, input.metadata, Config); + + if (searchResponse.erroneous) { + const error = handleSearchError(searchResponse); + errorResponseList.push(handleRtTfSingleEventError(input, error, {})); + } else { + transformedResponseToBeBatched.deletionData.push(...searchResponse.message); + transformedResponseToBeBatched.deletionSuccessMetadata.push(input.metadata); + } +}; + +/** + * Process the input message based on the specified action. + * If the 'fields' in the input message are empty, an error is generated. + * Determines whether to handle an upsert operation or a deletion operation based on the action. + * + * @param {Object} input - The input message containing the fields. + * @param {string} action - The action to be performed ('insert', 'update', or other). + * @param {string} operationModuleType - The type of operation module. + * @param {Object} Config - The configuration object. + * @param {Object} transformedResponseToBeBatched - The object to store transformed responses. + * @param {Array} errorResponseList - The list to store error responses. + */ +const processInput = async ( + input, + action, + operationModuleType, + Config, + transformedResponseToBeBatched, + errorResponseList, +) => { + const { fields } = input.message; + + if (isEmptyObject(fields)) { + const emptyFieldsError = new InstrumentationError('`fields` cannot be empty'); + errorResponseList.push(handleRtTfSingleEventError(input, emptyFieldsError, {})); + return; + } + + if (action === 'insert' || action === 'update') { + await handleUpsert( + input, + fields, + operationModuleType, + Config, + transformedResponseToBeBatched, + errorResponseList, + ); + } else { + await handleDeletion(input, fields, Config, transformedResponseToBeBatched, errorResponseList); + } +}; + +/** + * Appends success responses to the main response array. + * + * @param {Array} response - The main response array to which success responses will be appended. + * @param {Array} responseArray - An array of batched responses. + * @param {Array} metadataChunks - An array containing metadata chunks. + * @param {string} destination - The destination for the success responses. + */ +const appendSuccessResponses = (response, responseArray, metadataChunks, destination) => { + responseArray.forEach((batchedResponse, index) => { + response.push( + getSuccessRespEvents(batchedResponse, metadataChunks.items[index], destination, true), + ); + }); +}; + +/** + * Process multiple record inputs for a destination. + * + * @param {Array} inputs - The array of record inputs to be processed. + * @param {Object} destination - The destination object containing configuration. + * @returns {Array} - An array of responses after processing the record inputs. + */ +const processRecordInputs = async (inputs, destination) => { + if (!inputs || inputs.length === 0) { + return []; + } + + const response = []; + const errorResponseList = []; + const { Config } = destination; + const { action } = inputs[0].message; + + const transformedResponseToBeBatched = { + upsertData: [], + upsertSuccessMetadata: [], + deletionSuccessMetadata: [], + deletionData: [], + }; + + const { operationModuleType, identifierType, upsertEndPoint } = deduceModuleInfo(inputs, Config); + + validateConfigurationIssue(Config, operationModuleType, action); + + await Promise.all( + inputs.map((input) => + processInput( + input, + action, + operationModuleType, + Config, + transformedResponseToBeBatched, + errorResponseList, + ), + ), + ); + + const { + upsertResponseArray, + upsertmetadataChunks, + deletionResponseArray, + deletionmetadataChunks, + } = batchResponseBuilder( + transformedResponseToBeBatched, + Config, + identifierType, + operationModuleType, + upsertEndPoint, + action, + ); + + if (upsertResponseArray.length === 0 && deletionResponseArray.length === 0) { + return errorResponseList; + } + + appendSuccessResponses(response, upsertResponseArray, upsertmetadataChunks, destination); + appendSuccessResponses(response, deletionResponseArray, deletionmetadataChunks, destination); + + return [...response, ...errorResponseList]; +}; + +module.exports = { processRecordInputs }; diff --git a/src/cdk/v2/destinations/zoho/utils.js b/src/cdk/v2/destinations/zoho/utils.js new file mode 100644 index 0000000000..8b170d2b82 --- /dev/null +++ b/src/cdk/v2/destinations/zoho/utils.js @@ -0,0 +1,168 @@ +const { + MappedToDestinationKey, + getHashFromArray, + isDefinedAndNotNull, + ConfigurationError, +} = require('@rudderstack/integrations-lib'); +const get = require('get-value'); +const { getDestinationExternalIDInfoForRetl, isHttpStatusSuccess } = require('../../../../v0/util'); +const zohoConfig = require('./config'); +const { handleHttpRequest } = require('../../../../adapters/network'); + +const deduceModuleInfo = (inputs, Config) => { + const singleRecordInput = inputs[0].message; + const operationModuleInfo = {}; + const mappedToDestination = get(singleRecordInput, MappedToDestinationKey); + if (mappedToDestination) { + const { objectType, identifierType } = getDestinationExternalIDInfoForRetl( + singleRecordInput, + 'ZOHO', + ); + operationModuleInfo.operationModuleType = objectType; + operationModuleInfo.upsertEndPoint = zohoConfig + .COMMON_RECORD_ENDPOINT(Config.region) + .replace('moduleType', objectType); + operationModuleInfo.identifierType = identifierType; + } + return operationModuleInfo; +}; + +// eslint-disable-next-line consistent-return +function validatePresenceOfMandatoryProperties(objectName, object) { + if (zohoConfig.MODULE_MANDATORY_FIELD_CONFIG.hasOwnProperty(objectName)) { + const requiredFields = zohoConfig.MODULE_MANDATORY_FIELD_CONFIG[objectName]; + const missingFields = requiredFields.filter((field) => !object.hasOwnProperty(field)) || []; + return { status: missingFields.length > 0, missingField: missingFields }; + } + // No mandatory check performed for custom objects +} + +const formatMultiSelectFields = (config, fields) => { + // Convert multiSelectFieldLevelDecision array into a hash map for quick lookups + const multiSelectFields = getHashFromArray( + config.multiSelectFieldLevelDecision, + 'from', + 'to', + false, + ); + + Object.keys(fields).forEach((eachFieldKey) => { + if (multiSelectFields.hasOwnProperty(eachFieldKey)) { + // eslint-disable-next-line no-param-reassign + fields[eachFieldKey] = [fields[eachFieldKey]]; + } + }); + return fields; +}; + +// Utility to handle duplicate check +const handleDuplicateCheck = (addDefaultDuplicateCheck, identifierType, operationModuleType) => { + let duplicateCheckFields = [identifierType]; + + if (addDefaultDuplicateCheck) { + const moduleDuplicateCheckField = + zohoConfig.MODULE_WISE_DUPLICATE_CHECK_FIELD[operationModuleType]; + + if (isDefinedAndNotNull(moduleDuplicateCheckField)) { + duplicateCheckFields = [...moduleDuplicateCheckField]; + duplicateCheckFields.unshift(identifierType); + } else { + duplicateCheckFields.push('Name'); // user chosen duplicate field always carries higher priority + } + } + + return [...new Set(duplicateCheckFields)]; +}; + +function escapeAndEncode(value) { + return encodeURIComponent(value.replace(/([(),\\])/g, '\\$1')); +} + +function transformToURLParams(fields, Config) { + const criteria = Object.entries(fields) + .map(([key, value]) => `(${key}:equals:${escapeAndEncode(value)})`) + .join('and'); + + const dataCenter = Config.region; + const regionBasedEndPoint = zohoConfig.DATA_CENTRE_BASE_ENDPOINTS_MAP[dataCenter]; + + return `${regionBasedEndPoint}/crm/v6/Leads/search?criteria=${criteria}`; +} + +// ref : https://www.zoho.com/crm/developer/docs/api/v6/search-records.html +const searchRecordId = async (fields, metadata, Config) => { + const searchURL = transformToURLParams(fields, Config); + const searchResult = await handleHttpRequest( + 'get', + searchURL, + { + headers: { + Authorization: `Zoho-oauthtoken ${metadata.secret.accessToken}`, + }, + }, + { + destType: 'zoho', + feature: 'deleteRecords', + requestMethod: 'GET', + endpointPath: 'crm/v6/Leads/search?criteria=', + module: 'router', + }, + ); + if (!isHttpStatusSuccess(searchResult.processedResponse.status)) { + return { + erroneous: true, + message: searchResult.processedResponse.response, + }; + } + if (searchResult.processedResponse.status === 204) { + return { + erroneous: true, + message: 'No contact is found with record details', + }; + } + const recordIds = searchResult.processedResponse.response.data.map((record) => record.id); + return { + erroneous: false, + message: recordIds, + }; +}; + +// ref : https://www.zoho.com/crm/developer/docs/api/v6/upsert-records.html#:~:text=The%20trigger%20input%20can%20be%20workflow%2C%20approval%2C%20or%20blueprint.%20If%20the%20trigger%20is%20not%20mentioned%2C%20the%20workflows%2C%20approvals%20and%20blueprints%20related%20to%20the%20API%20will%20get%20executed.%20Enter%20the%20trigger%20value%20as%20%5B%5D%20to%20not%20execute%20the%20workflows. +const calculateTrigger = (trigger) => { + if (trigger === 'Default') { + return null; + } + if (trigger === 'None') { + return []; + } + return [trigger]; +}; + +const validateConfigurationIssue = (Config, operationModuleType, action) => { + const hashMapMultiselect = getHashFromArray( + Config.multiSelectFieldLevelDecision, + 'from', + 'to', + false, + ); + if ( + Object.keys(hashMapMultiselect).length > 0 && + Config.module !== operationModuleType && + action !== 'delete' + ) { + throw new ConfigurationError( + 'Object Chosen in Visual Data Mapper is not consistent with Module type selected in destination configuration. Aborting Events.', + ); + } +}; + +module.exports = { + deduceModuleInfo, + validatePresenceOfMandatoryProperties, + formatMultiSelectFields, + handleDuplicateCheck, + searchRecordId, + transformToURLParams, + calculateTrigger, + validateConfigurationIssue, +}; diff --git a/src/cdk/v2/destinations/zoho/utils.test.js b/src/cdk/v2/destinations/zoho/utils.test.js new file mode 100644 index 0000000000..332a408695 --- /dev/null +++ b/src/cdk/v2/destinations/zoho/utils.test.js @@ -0,0 +1,245 @@ +const { + handleDuplicateCheck, + deduceModuleInfo, + validatePresenceOfMandatoryProperties, + formatMultiSelectFields, + validateConfigurationIssue, +} = require('./utils'); + +const { ConfigurationError } = require('@rudderstack/integrations-lib'); + +describe('handleDuplicateCheck', () => { + // Returns identifierType when addDefaultDuplicateCheck is false + it('should return identifierType when addDefaultDuplicateCheck is false', () => { + const identifierType = 'email'; + const addDefaultDuplicateCheck = false; + const operationModuleType = 'Leads'; + const moduleWiseDuplicateCheckField = {}; + + const result = handleDuplicateCheck( + addDefaultDuplicateCheck, + identifierType, + operationModuleType, + moduleWiseDuplicateCheckField, + ); + + expect(result).toEqual([identifierType]); + }); + + it('Handles valid operationModuleType and already included identifierType', () => { + const identifierType = 'Email'; + const addDefaultDuplicateCheck = true; + const operationModuleType = 'Leads'; + + const result = handleDuplicateCheck( + addDefaultDuplicateCheck, + identifierType, + operationModuleType, + ); + + expect(result).toEqual(['Email']); + }); + + // Returns identifierType and 'Name' when addDefaultDuplicateCheck is true and moduleDuplicateCheckField is not defined + it("should return identifierType and 'Name' when addDefaultDuplicateCheck is true and moduleDuplicateCheckField is not defined", () => { + const identifierType = 'id'; + const operationModuleType = 'type3'; + const addDefaultDuplicateCheck = true; + + const result = handleDuplicateCheck( + addDefaultDuplicateCheck, + identifierType, + operationModuleType, + ); + + expect(result).toEqual(['id', 'Name']); + }); + + // Handles null values in moduleWiseDuplicateCheckField + it('should handle null values in moduleWiseDuplicateCheckField', () => { + const addDefaultDuplicateCheck = true; + const identifierType = 'Identifier'; + const operationModuleType = 'type1'; + + const result = handleDuplicateCheck( + addDefaultDuplicateCheck, + identifierType, + operationModuleType, + ); + + expect(result).toEqual(['Identifier', 'Name']); + }); +}); + +describe('deduceModuleInfo', () => { + const Config = { region: 'US' }; + + it('should return empty object when mappedToDestination is not present', () => { + const inputs = [{}]; + const result = deduceModuleInfo(inputs, Config); + expect(result).toEqual({}); + }); + + it('should return operationModuleInfo when mappedToDestination is present', () => { + const inputs = [ + { + message: { + context: { + externalId: [{ type: 'ZOHO-Leads', id: '12345', identifierType: 'Email' }], + mappedToDestination: true, + }, + }, + }, + ]; + + const result = deduceModuleInfo(inputs, Config); + expect(result).toEqual({ + operationModuleType: 'Leads', + upsertEndPoint: 'https://www.zohoapis.com/crm/v6/Leads', + identifierType: 'Email', + }); + }); + + it('should handle different regions in config', () => { + const inputs = [ + { + message: { + context: { + externalId: [{ type: 'ZOHO-Leads', id: '12345', identifierType: 'Email' }], + mappedToDestination: 'true', + }, + }, + }, + ]; + const Config = { region: 'EU' }; + + const result = deduceModuleInfo(inputs, Config); + expect(result).toEqual({ + operationModuleType: 'Leads', + upsertEndPoint: 'https://www.zohoapis.eu/crm/v6/Leads', + identifierType: 'Email', + }); + }); +}); + +describe('validatePresenceOfMandatoryProperties', () => { + it('should not throw an error if the object has all required fields', () => { + const objectName = 'Leads'; + const object = { Last_Name: 'Doe' }; + + expect(() => validatePresenceOfMandatoryProperties(objectName, object)).not.toThrow(); + }); + + it('should not throw an error if the objectName is not in MODULE_MANDATORY_FIELD_CONFIG', () => { + const objectName = 'CustomObject'; + const object = { Some_Field: 'Some Value' }; + + expect(() => validatePresenceOfMandatoryProperties(objectName, object)).not.toThrow(); + }); + + it('should throw an error if the object is missing multiple required fields', () => { + const objectName = 'Deals'; + const object = { Deal_Name: 'Big Deal' }; + const output = validatePresenceOfMandatoryProperties(objectName, object); + expect(output).toEqual({ + missingField: ['Stage', 'Pipeline'], + status: true, + }); + }); + + it('should not throw an error if the object has all required fields for Deals', () => { + const objectName = 'Deals'; + const object = { Deal_Name: 'Big Deal', Stage: 'Negotiation', Pipeline: 'Sales' }; + + expect(() => validatePresenceOfMandatoryProperties(objectName, object)).not.toThrow(); + }); +}); + +describe('validateConfigurationIssue', () => { + test('should throw ConfigurationError when hashMapMultiselect is not empty, Config.module is different from operationModuleType, and action is not delete', () => { + const Config = { + multiSelectFieldLevelDecision: [{ from: 'field1', to: 'true' }], + module: 'moduleA', + }; + const operationModuleType = 'moduleB'; + const action = 'create'; + + expect(() => validateConfigurationIssue(Config, operationModuleType, action)).toThrow( + ConfigurationError, + ); + expect(() => validateConfigurationIssue(Config, operationModuleType, action)).toThrow( + 'Object Chosen in Visual Data Mapper is not consistent with Module type selected in destination configuration. Aborting Events.', + ); + }); + + test('should not throw an error when hashMapMultiselect is not empty, Config.module is the same as operationModuleType, and action is not delete', () => { + const Config = { + multiSelectFieldLevelDecision: [{ from: 'field1', to: 'true' }], + module: 'moduleA', + }; + const operationModuleType = 'moduleA'; + const action = 'create'; + + expect(() => validateConfigurationIssue(Config, operationModuleType, action)).not.toThrow(); + }); + + test('should not throw an error when hashMapMultiselect is empty, Config.module is different from operationModuleType, and action is not delete', () => { + const Config = { + multiSelectFieldLevelDecision: [], + module: 'moduleA', + }; + const operationModuleType = 'moduleB'; + const action = 'create'; + + expect(() => validateConfigurationIssue(Config, operationModuleType, action)).not.toThrow(); + }); + + test('should not throw an error when hashMapMultiselect is empty, Config.module is the same as operationModuleType, and action is not delete', () => { + const Config = { + multiSelectFieldLevelDecision: [], + module: 'moduleA', + }; + const operationModuleType = 'moduleA'; + const action = 'create'; + + expect(() => validateConfigurationIssue(Config, operationModuleType, action)).not.toThrow(); + }); + + test('should not throw an error when multiSelectFieldLevelDecision has entries without from key', () => { + const Config = { + multiSelectFieldLevelDecision: [{ to: 'true' }], + module: 'moduleA', + }; + const operationModuleType = 'moduleB'; + const action = 'create'; + + expect(() => validateConfigurationIssue(Config, operationModuleType, action)).not.toThrow(); + }); + + test('should throw ConfigurationError when multiSelectFieldLevelDecision has mixed case from keys, Config.module is different from operationModuleType, and action is not delete', () => { + const Config = { + multiSelectFieldLevelDecision: [ + { from: 'FIELD1', to: 'true' }, + { from: 'field2', to: 'false' }, + ], + module: 'moduleA', + }; + const operationModuleType = 'moduleB'; + const action = 'create'; + + expect(() => validateConfigurationIssue(Config, operationModuleType, action)).toThrow( + ConfigurationError, + ); + }); + + test('should not throw an error when hashMapMultiselect is not empty, Config.module is different from operationModuleType, and action is delete', () => { + const Config = { + multiSelectFieldLevelDecision: [{ from: 'field1', to: 'true' }], + module: 'moduleA', + }; + const operationModuleType = 'moduleB'; + const action = 'delete'; + + expect(() => validateConfigurationIssue(Config, operationModuleType, action)).not.toThrow(); + }); +}); diff --git a/src/features.json b/src/features.json index 78737193a8..94e36a2416 100644 --- a/src/features.json +++ b/src/features.json @@ -75,6 +75,7 @@ "KODDI": true, "WUNDERKIND": true, "CLICKSEND": true, + "ZOHO": true, "CORDIAL": true }, "regulations": [ diff --git a/src/v1/destinations/zoho/networkHandler.js b/src/v1/destinations/zoho/networkHandler.js new file mode 100644 index 0000000000..2ceb0bbdf3 --- /dev/null +++ b/src/v1/destinations/zoho/networkHandler.js @@ -0,0 +1,141 @@ +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); +const { proxyRequest, prepareProxyRequest } = require('../../../adapters/network'); +const { + processAxiosResponse, + getDynamicErrorType, +} = require('../../../adapters/utils/networkUtils'); +const { isHttpStatusSuccess, getAuthErrCategoryFromStCode } = require('../../../v0/util/index'); +const tags = require('../../../v0/util/tags'); + +/** + * upsert response : + + { + "data": [ + { + "code": "INVALID_DATA", + "details": { + "expected_data_type": "integer", + "api_name": "No_of_Employees", + "json_path": "$.data[0].No_of_Employees" + }, + "message": "invalid data", + "status": "error" + }, + { + "code": "SUCCESS", + "duplicate_field": "Email", + "action": "update", + "details": { + "Modified_Time": "2024-07-14T10:54:15+05:30", + "Modified_By": { + "name": "dummy user", + "id": "724445000000323001" + }, + "Created_Time": "2024-07-01T21:25:36+05:30", + "id": "724445000000349039", + "Created_By": { + "name": "dummy user", + "id": "724445000000323001" + } + }, + "message": "record updated", + "status": "success" + } + ] +} + +* delete response : + + { + "data": [ + { + "code": "SUCCESS", + "details": { + "id": "724445000000445001" + }, + "message": "record deleted", + "status": "success" + }, + { + "code": "INVALID_DATA", + "details": { + "id": "724445000000323001" + }, + "message": "record not deleted", + "status": "error" + } + ] +} + */ + +const checkIfEventIsAbortableAndExtractErrorMessage = (element) => { + if (element.status === 'success') { + return { isAbortable: false, errorMsg: '' }; + } + + const errorMsg = `message: ${element.messaege} ${JSON.stringify(element.details)}`; + return { isAbortable: true, errorMsg }; +}; + +const responseHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata } = responseParams; + + const message = '[ZOHO Response V1 Handler] - Request Processed Successfully'; + const responseWithIndividualEvents = []; + const { response, status } = destinationResponse; + if (isHttpStatusSuccess(status)) { + // check for Partial Event failures and Successes + const { data } = response; + data.forEach((event, idx) => { + const proxyOutput = { + statusCode: 200, + metadata: rudderJobMetadata[idx], + error: 'success', + }; + // update status of partial event if abortable + const { isAbortable, errorMsg } = checkIfEventIsAbortableAndExtractErrorMessage(event); + if (isAbortable) { + proxyOutput.statusCode = 400; + proxyOutput.error = errorMsg; + } + responseWithIndividualEvents.push(proxyOutput); + }); + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; + } + + if (response?.code === 'INVALID_TOKEN') { + throw new TransformerProxyError( + `Zoho: Error transformer proxy v1 during Zoho response transformation. ${response.message}`, + 500, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(500), + }, + destinationResponse, + getAuthErrCategoryFromStCode(status), + response.message, + ); + } + throw new TransformerProxyError( + `ZOHO: Error encountered in transformer proxy V1`, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + '', + responseWithIndividualEvents, + ); +}; +function networkHandler() { + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.prepareProxy = prepareProxyRequest; + this.responseHandler = responseHandler; +} +module.exports = { networkHandler }; diff --git a/test/integrations/component.test.ts b/test/integrations/component.test.ts index 388c283c61..d4c109c55c 100644 --- a/test/integrations/component.test.ts +++ b/test/integrations/component.test.ts @@ -28,7 +28,7 @@ import _ from 'lodash'; // To run single destination test cases // npm run test:ts -- component --destination=adobe_analytics // npm run test:ts -- component --destination=adobe_analytics --feature=router -// npm run test:ts -- component --destination=adobe_analytics --feature=router --index=0 +// npm run test:ts -- component --destination=adobe_analytics --feature=dataDelivery --index=0 // Use below command to generate mocks // npm run test:ts -- component --destination=zendesk --generate=true @@ -101,6 +101,8 @@ if (!opts.generate || opts.generate === 'false') { // END const rootDir = __dirname; +console.log('rootDir', rootDir); +console.log('opts', opts); const allTestDataFilePaths = getTestDataFilePaths(rootDir, opts); const DEFAULT_VERSION = 'v0'; diff --git a/test/integrations/destinations/zoho/common.ts b/test/integrations/destinations/zoho/common.ts new file mode 100644 index 0000000000..bea4437e6f --- /dev/null +++ b/test/integrations/destinations/zoho/common.ts @@ -0,0 +1,334 @@ +import { Destination } from '../../../../src/types'; + +const destType = 'zoho'; +const destTypeInUpperCase = 'ZOHO'; +const advertiserId = 'test-advertiser-id'; +const dataProviderId = 'rudderstack'; +const segmentName = 'test-segment'; +const leadUpsertEndpoint = 'https://www.zohoapis.in/crm/v6/Leads/upsert'; + +const deletionPayload1 = { + action: 'delete', + context: { + externalId: [ + { + type: 'ZOHO-Leads', + identifierType: 'email', + }, + ], + mappedToDestination: 'true', + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + Email: 'tobedeleted@gmail.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + type: 'record', +}; + +const commonDeletionDestConfig: Destination = { + ID: '345', + Name: 'Test', + Enabled: true, + WorkspaceID: '', + Transformations: [], + DestinationDefinition: { + ID: '345', + Name: 'Test', + DisplayName: 'ZOHO', + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + region: 'IN', + module: 'Leads', + trigger: 'None', + addDefaultDuplicateCheck: true, + multiSelectFieldLevelDecision: [ + { + from: 'multi-language', + to: 'true', + }, + { + from: 'multi class', + to: 'false', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, +}; + +const upsertPayload1 = { + action: 'insert', + context: { + externalId: [ + { + type: 'ZOHO-Leads', + identifierType: 'email', + }, + ], + mappedToDestination: 'true', + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + type: 'record', +}; + +const upsertPayload2 = { + action: 'insert', + context: { + externalId: [ + { + type: 'ZOHO-Leads', + identifierType: 'email', + }, + ], + mappedToDestination: 'true', + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + 'multi-language': 'Bengali', + }, + type: 'record', +}; + +const upsertPayload3 = { + action: 'insert', + context: { + externalId: [ + { + type: 'ZOHO-Leads', + identifierType: 'Email', + }, + ], + mappedToDestination: 'true', + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + type: 'record', +}; + +const commonUpsertDestConfig: Destination = { + ID: '345', + Name: 'Test', + Enabled: true, + WorkspaceID: '', + Transformations: [], + DestinationDefinition: { + ID: '345', + Name: 'Test', + DisplayName: 'ZOHO', + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + region: 'US', + module: 'Leads', + trigger: 'workflow', + addDefaultDuplicateCheck: true, + multiSelectFieldLevelDecision: [ + { + from: 'multi-language', + to: 'true', + }, + { + from: 'multi class', + to: 'false', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, +}; + +const commonUpsertDestConfig2: Destination = { + ID: '345', + Name: 'Test', + Enabled: true, + WorkspaceID: '', + Transformations: [], + DestinationDefinition: { + ID: '345', + Name: 'Test', + DisplayName: 'ZOHO', + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + region: 'US', + module: 'Leads', + trigger: 'None', + addDefaultDuplicateCheck: true, + multiSelectFieldLevelDecision: [ + { + from: 'multi-language', + to: 'true', + }, + { + from: 'multi class', + to: 'false', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, +}; + +const commonUpsertDestConfig2CustomModule: Destination = { + ID: '345', + Name: 'Test', + Enabled: true, + WorkspaceID: '', + Transformations: [], + DestinationDefinition: { + ID: '345', + Name: 'Test', + DisplayName: 'ZOHO', + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + region: 'US', + module: 'CUSTOM', + trigger: 'None', + addDefaultDuplicateCheck: true, + multiSelectFieldLevelDecision: [ + { + from: 'multi-language', + to: 'true', + }, + { + from: 'multi class', + to: 'false', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, +}; + +const commonUpsertDestConfig3: Destination = { + ID: '345', + Name: 'Test', + Enabled: true, + WorkspaceID: '', + Transformations: [], + DestinationDefinition: { + ID: '345', + Name: 'Test', + DisplayName: 'ZOHO', + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + region: 'US', + module: 'Leads', + trigger: 'workflow', + addDefaultDuplicateCheck: true, + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, +}; + +const commonOutput1 = { + duplicate_check_fields: ['Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + ], + $append_values: {}, + trigger: ['workflow'], +}; + +export { + destType, + destTypeInUpperCase, + advertiserId, + dataProviderId, + segmentName, + leadUpsertEndpoint, + deletionPayload1, + commonDeletionDestConfig, + upsertPayload1, + upsertPayload2, + upsertPayload3, + commonUpsertDestConfig, + commonUpsertDestConfig2, + commonOutput1, + commonUpsertDestConfig3, + commonUpsertDestConfig2CustomModule, +}; diff --git a/test/integrations/destinations/zoho/dataDelivery/business.ts b/test/integrations/destinations/zoho/dataDelivery/business.ts new file mode 100644 index 0000000000..89c3ca214b --- /dev/null +++ b/test/integrations/destinations/zoho/dataDelivery/business.ts @@ -0,0 +1,401 @@ +import { generateMetadata, generateProxyV1Payload } from '../../../testUtils'; +import { ProxyV1TestData } from '../../../testTypes'; + +export const headerBlockWithCorrectAccessToken = { + 'Content-Type': 'application/json', + Authorization: 'Zoho-oauthtoken dummy-key', +}; + +export const contactPayload = { + duplicate_check_fields: ['Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + ], + $append_values: {}, + trigger: ['workflow'], +}; + +export const statTags = { + destType: 'ZOHO', + errorCategory: 'network', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', +}; + +export const metadata = [ + { + jobId: 1, + attemptNum: 1, + userId: 'default-userId', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + sourceId: 'default-sourceId', + secret: { + accessToken: 'default-accessToken', + }, + dontBatch: false, + }, + { + jobId: 2, + attemptNum: 1, + userId: 'default-userId', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + sourceId: 'default-sourceId', + secret: { + accessToken: 'default-accessToken', + }, + dontBatch: false, + }, + { + jobId: 3, + attemptNum: 1, + userId: 'default-userId', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + sourceId: 'default-sourceId', + secret: { + accessToken: 'default-accessToken', + }, + dontBatch: false, + }, +]; + +export const singleMetadata = [ + { + jobId: 1, + attemptNum: 1, + userId: 'default-userId', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + sourceId: 'default-sourceId', + secret: { + accessToken: 'default-accessToken', + }, + dontBatch: false, + }, +]; + +const commonRecordParameters = { + method: 'POST', + headers: headerBlockWithCorrectAccessToken, + JSON: { ...contactPayload }, +}; + +const commonRecordParametersWithWrongToken = { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: 'Zoho-oauthtoken wrong-token' }, + JSON: { + duplicate_check_fields: ['Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + ], + $append_values: {}, + trigger: ['workflow'], + }, +}; + +const multiContactPayload = { + duplicate_check_fields: ['Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + { + Random: 'subscribed@eewrfrd.com', + }, + ], + $append_values: {}, + trigger: ['workflow'], +}; + +const commonMultiRecordParameters = { + method: 'POST', + headers: headerBlockWithCorrectAccessToken, + JSON: { ...multiContactPayload }, +}; + +export const testScenariosForV1API: ProxyV1TestData[] = [ + { + id: 'zoho_v1_scenario_1', + name: 'zoho', + description: 'Upserting Leads successfully', + successCriteria: 'Should return 200 and success', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://www.zohoapis.in/crm/v6/Leads/upsert', + ...commonRecordParameters, + }, + singleMetadata, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[ZOHO Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + data: [ + { + code: 'SUCCESS', + duplicate_field: null, + action: 'insert', + details: { + Modified_Time: '2024-07-16T09:39:27+05:30', + Modified_By: { + name: 'Dummy-User', + id: '724445000000323001', + }, + Created_Time: '2024-07-16T09:39:27+05:30', + id: '724445000000424003', + Created_By: { + name: 'Dummy-User', + id: '724445000000323001', + }, + $approval_state: 'approved', + }, + message: 'record added', + status: 'success', + }, + ], + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + ], + }, + }, + }, + }, + }, + { + id: 'zoho_v1_scenario_2', + name: 'zoho', + description: 'Trying to upsert in wrong module name', + successCriteria: 'Should return 400 and should be aborted', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://www.zohoapis.in/crm/v6/Wrong/upsert', + ...commonRecordParameters, + }, + singleMetadata, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 400, + statTags, + message: 'ZOHO: Error encountered in transformer proxy V1', + response: [ + { + statusCode: 400, + metadata: generateMetadata(1), + error: + '{"code":"INVALID_MODULE","details":{"resource_path_index":0},"message":"the module name given seems to be invalid","status":"error"}', + }, + ], + }, + }, + }, + }, + }, + { + id: 'zoho_v1_scenario_3', + name: 'zoho', + description: 'Trying to upsert using invalid access token', + successCriteria: 'Should return 500 and try for refreshed token', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://www.zohoapis.in/crm/v6/Leads/upsert', + ...commonRecordParametersWithWrongToken, + }, + singleMetadata, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 500, + body: { + output: { + status: 500, + statTags: { ...statTags, errorType: 'retryable' }, + message: + 'Zoho: Error transformer proxy v1 during Zoho response transformation. invalid oauth token', + authErrorCategory: 'REFRESH_TOKEN', + response: [ + { + error: + '{"code":"INVALID_TOKEN","details":{},"message":"invalid oauth token","status":"error"}', + statusCode: 500, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + }, + }, + { + id: 'zoho_v1_scenario_4', + name: 'zoho', + description: 'testing partial failure', + successCriteria: 'Should return 200 and success for successful and 400 for failed payloads', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://www.zohoapis.in/crm/v6/Leads/upsert', + ...commonMultiRecordParameters, + }, + metadata, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[ZOHO Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + data: [ + { + code: 'SUCCESS', + duplicate_field: 'Email', + action: 'update', + details: { + Modified_Time: '2024-07-16T15:01:02+05:30', + Modified_By: { + name: 'dummy-user', + id: '724445000000323001', + }, + Created_Time: '2024-07-16T09:39:27+05:30', + id: '724445000000424003', + Created_By: { + name: 'dummy-user', + id: '724445000000323001', + }, + }, + message: 'record updated', + status: 'success', + }, + { + code: 'SUCCESS', + duplicate_field: 'Email', + action: 'update', + details: { + Modified_Time: '2024-07-16T15:01:02+05:30', + Modified_By: { + name: 'dummy-user', + id: '724445000000323001', + }, + Created_Time: '2024-07-16T09:39:27+05:30', + id: '724445000000424003', + Created_By: { + name: 'dummy-user', + id: '724445000000323001', + }, + }, + message: 'record updated', + status: 'success', + }, + { + code: 'MANDATORY_NOT_FOUND', + details: { + api_name: 'Last_Name', + json_path: '$.data[2].Last_Name', + }, + message: 'required field not found', + status: 'error', + }, + ], + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + { + error: 'success', + metadata: generateMetadata(2), + statusCode: 200, + }, + { + error: + 'message: undefined {"api_name":"Last_Name","json_path":"$.data[2].Last_Name"}', + metadata: generateMetadata(3), + statusCode: 400, + }, + ], + }, + }, + }, + }, + }, +]; + +export const data = [...testScenariosForV1API]; diff --git a/test/integrations/destinations/zoho/dataDelivery/data.ts b/test/integrations/destinations/zoho/dataDelivery/data.ts new file mode 100644 index 0000000000..fc969bb8e1 --- /dev/null +++ b/test/integrations/destinations/zoho/dataDelivery/data.ts @@ -0,0 +1,3 @@ +import { testScenariosForV1API } from './business'; + +export const data = [...testScenariosForV1API]; diff --git a/test/integrations/destinations/zoho/mocks.ts b/test/integrations/destinations/zoho/mocks.ts new file mode 100644 index 0000000000..1e4c7d18c7 --- /dev/null +++ b/test/integrations/destinations/zoho/mocks.ts @@ -0,0 +1,5 @@ +import config from '../../../../src/cdk/v2/destinations/zoho/config'; + +export const defaultMockFns = () => { + jest.replaceProperty(config, 'MAX_BATCH_SIZE', 2); +}; diff --git a/test/integrations/destinations/zoho/network.ts b/test/integrations/destinations/zoho/network.ts new file mode 100644 index 0000000000..b37a56d123 --- /dev/null +++ b/test/integrations/destinations/zoho/network.ts @@ -0,0 +1,421 @@ +import { destType } from './common'; + +export const networkCallsData = [ + { + httpReq: { + url: 'https://www.zohoapis.in/crm/v6/Leads/upsert', + data: { + duplicate_check_fields: ['Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + ], + $append_values: {}, + trigger: ['workflow'], + }, + params: { destination: destType }, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Zoho-oauthtoken dummy-key', + }, + method: 'POST', + }, + httpRes: { + data: { + data: [ + { + code: 'SUCCESS', + duplicate_field: null, + action: 'insert', + details: { + Modified_Time: '2024-07-16T09:39:27+05:30', + Modified_By: { + name: 'Dummy-User', + id: '724445000000323001', + }, + Created_Time: '2024-07-16T09:39:27+05:30', + id: '724445000000424003', + Created_By: { + name: 'Dummy-User', + id: '724445000000323001', + }, + $approval_state: 'approved', + }, + message: 'record added', + status: 'success', + }, + ], + }, + status: 200, + statusText: 'OK', + }, + }, + { + httpReq: { + url: 'https://www.zohoapis.in/crm/v6/Wrong/upsert', + data: { + duplicate_check_fields: ['Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + ], + $append_values: {}, + trigger: ['workflow'], + }, + params: { destination: destType }, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Zoho-oauthtoken dummy-key', + }, + method: 'POST', + }, + httpRes: { + data: { + code: 'INVALID_MODULE', + details: { + resource_path_index: 0, + }, + message: 'the module name given seems to be invalid', + status: 'error', + }, + status: 400, + statusText: 'Bad Request', + }, + }, + { + httpReq: { + url: 'https://www.zohoapis.in/crm/v6/Leads/upsert', + data: { + duplicate_check_fields: ['Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + ], + $append_values: {}, + trigger: ['workflow'], + }, + params: { destination: destType }, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Zoho-oauthtoken wrong-token', + }, + method: 'POST', + }, + httpRes: { + data: { + code: 'INVALID_TOKEN', + details: {}, + message: 'invalid oauth token', + status: 'error', + }, + status: 401, + statusText: 'Bad Request', + }, + }, + { + httpReq: { + url: 'https://www.zohoapis.in/crm/v6/Leads/upsert', + data: { + duplicate_check_fields: ['Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + { + Random: 'subscribed@eewrfrd.com', + }, + ], + $append_values: {}, + trigger: ['workflow'], + }, + params: { destination: destType }, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Zoho-oauthtoken dummy-key', + }, + method: 'POST', + }, + httpRes: { + data: { + data: [ + { + code: 'SUCCESS', + duplicate_field: 'Email', + action: 'update', + details: { + Modified_Time: '2024-07-16T15:01:02+05:30', + Modified_By: { + name: 'dummy-user', + id: '724445000000323001', + }, + Created_Time: '2024-07-16T09:39:27+05:30', + id: '724445000000424003', + Created_By: { + name: 'dummy-user', + id: '724445000000323001', + }, + }, + message: 'record updated', + status: 'success', + }, + { + code: 'SUCCESS', + duplicate_field: 'Email', + action: 'update', + details: { + Modified_Time: '2024-07-16T15:01:02+05:30', + Modified_By: { + name: 'dummy-user', + id: '724445000000323001', + }, + Created_Time: '2024-07-16T09:39:27+05:30', + id: '724445000000424003', + Created_By: { + name: 'dummy-user', + id: '724445000000323001', + }, + }, + message: 'record updated', + status: 'success', + }, + { + code: 'MANDATORY_NOT_FOUND', + details: { + api_name: 'Last_Name', + json_path: '$.data[2].Last_Name', + }, + message: 'required field not found', + status: 'error', + }, + ], + // }, + }, + status: 200, + statusText: 'OK', + }, + }, + { + httpReq: { + url: 'https://www.zohoapis.in/crm/v6/Leads/search?criteria=(Email:equals:tobedeleted3%40gmail.com)and(First_Name:equals:subcribed3)and(Last_Name:equals:%20User3)', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + method: 'GET', + }, + httpRes: { + data: { + data: '', + }, + status: 204, + statusText: 'OK', + }, + }, + { + httpReq: { + url: 'https://www.zohoapis.in/crm/v6/Leads/search?criteria=(Email:equals:tobedeleted%40gmail.com)and(First_Name:equals:subcribed)and(Last_Name:equals:%20User)', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + method: 'GET', + }, + httpRes: { + data: { + data: [ + { + Owner: { + name: 'dummy-user', + id: '724445000000323001', + email: 'dummy@gmail.com', + }, + Company: null, + Email: 'tobedeleted@gmail.com', + $currency_symbol: '$', + $field_states: null, + $sharing_permission: 'full_access', + Last_Activity_Time: '2024-07-18T23:55:42+05:30', + Industry: null, + Unsubscribed_Mode: null, + $process_flow: false, + Street: null, + Zip_Code: null, + id: '', + $approval: { + delegate: false, + approve: false, + reject: false, + resubmit: false, + }, + Created_Time: '2024-07-18T19:34:50+05:30', + $editable: true, + City: null, + No_of_Employees: null, + Converted_Account: null, + State: null, + Country: null, + Created_By: { + name: 'dummy-user', + id: '724445000000323001', + email: 'dummy@gmail.com', + }, + $zia_owner_assignment: 'owner_recommendation_unavailable', + Annual_Revenue: null, + Secondary_Email: null, + Description: null, + Rating: null, + $review_process: { + approve: false, + reject: false, + resubmit: false, + }, + Website: null, + Twitter: null, + Salutation: null, + First_Name: 'subcribed', + Full_Name: 'subcribed User', + Lead_Status: null, + Record_Image: null, + Modified_By: { + name: 'dummy-user', + id: '724445000000323001', + email: 'dummy@gmail.com', + }, + Converted_Deal: null, + $review: null, + Lead_Conversion_Time: null, + Skype_ID: null, + Phone: null, + Email_Opt_Out: false, + $zia_visions: null, + Designation: null, + Modified_Time: '2024-07-18T23:55:42+05:30', + $converted_detail: {}, + Unsubscribed_Time: null, + Converted_Contact: null, + Mobile: null, + $orchestration: null, + Last_Name: 'User', + $in_merge: false, + Lead_Source: null, + Fax: null, + $approval_state: 'approved', + $pathfinder: null, + }, + ], + }, + status: 200, + statusText: 'OK', + }, + }, + { + httpReq: { + url: 'https://www.zohoapis.in/crm/v6/Leads/search?criteria=(Email:equals:tobedeleted2%40gmail.com)and(First_Name:equals:subcribed2)and(Last_Name:equals:%20User2)', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + method: 'GET', + }, + httpRes: { + data: { + data: [ + { + Owner: { + name: 'dummy-user', + id: '724445000000323001', + email: 'dummy@gmail.com', + }, + Company: null, + Email: 'tobedeleted2@gmail.com', + $currency_symbol: '$', + $field_states: null, + $sharing_permission: 'full_access', + Last_Activity_Time: '2024-07-18T23:55:42+05:30', + Industry: null, + Unsubscribed_Mode: null, + $process_flow: false, + Street: null, + Zip_Code: null, + id: '', + $approval: { + delegate: false, + approve: false, + reject: false, + resubmit: false, + }, + Created_Time: '2024-07-18T19:34:50+05:30', + $editable: true, + City: null, + No_of_Employees: null, + Converted_Account: null, + State: null, + Country: null, + Created_By: { + name: 'dummy-user', + id: '724445000000323001', + email: 'dummy@gmail.com', + }, + $zia_owner_assignment: 'owner_recommendation_unavailable', + Annual_Revenue: null, + Secondary_Email: null, + Description: null, + Rating: null, + $review_process: { + approve: false, + reject: false, + resubmit: false, + }, + Website: null, + Twitter: null, + Salutation: null, + First_Name: 'subcribed2', + Full_Name: 'subcribed2 User', + Lead_Status: null, + Record_Image: null, + Modified_By: { + name: 'dummy-user', + id: '724445000000323001', + email: 'dummy@gmail.com', + }, + Converted_Deal: null, + $review: null, + Lead_Conversion_Time: null, + Skype_ID: null, + Phone: null, + Email_Opt_Out: false, + $zia_visions: null, + Designation: null, + Modified_Time: '2024-07-18T23:55:42+05:30', + $converted_detail: {}, + Unsubscribed_Time: null, + Converted_Contact: null, + Mobile: null, + $orchestration: null, + Last_Name: 'User2', + $in_merge: false, + Lead_Source: null, + Fax: null, + $approval_state: 'approved', + $pathfinder: null, + }, + ], + }, + status: 200, + statusText: 'OK', + }, + }, +]; diff --git a/test/integrations/destinations/zoho/router/data.ts b/test/integrations/destinations/zoho/router/data.ts new file mode 100644 index 0000000000..7340fd18c8 --- /dev/null +++ b/test/integrations/destinations/zoho/router/data.ts @@ -0,0 +1,4 @@ +import { upsertData } from './upsert'; +import { deleteData } from './deletion'; + +export const data = [...upsertData, ...deleteData]; diff --git a/test/integrations/destinations/zoho/router/deletion.ts b/test/integrations/destinations/zoho/router/deletion.ts new file mode 100644 index 0000000000..5e922bc794 --- /dev/null +++ b/test/integrations/destinations/zoho/router/deletion.ts @@ -0,0 +1,249 @@ +import { defaultMockFns } from '../mocks'; +import { commonDeletionDestConfig, deletionPayload1, destType } from '../common'; + +export const deleteData = [ + { + name: destType, + id: 'zoho_deletion_1', + description: 'Happy flow record deletion with Leads module', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: deletionPayload1, + metadata: { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonDeletionDestConfig, + }, + { + message: { + action: 'delete', + context: { + externalId: [ + { + type: 'ZOHO-Leads', + identifierType: 'email', + }, + ], + mappedToDestination: 'true', + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + Email: 'tobedeleted2@gmail.com', + First_Name: 'subcribed2', + Last_Name: ' User2', + }, + type: 'record', + }, + metadata: { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonDeletionDestConfig, + }, + ], + destType, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://www.zohoapis.in/crm/v6/Leads?ids=,&wf_trigger=false', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonDeletionDestConfig, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, + { + name: destType, + id: 'zoho_deletion_2', + description: 'Batch containing already existing and non existing records for deletion', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: deletionPayload1, + metadata: { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonDeletionDestConfig, + }, + { + message: { + action: 'delete', + context: { + externalId: [ + { + type: 'ZOHO-Leads', + identifierType: 'email', + }, + ], + mappedToDestination: 'true', + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + Email: 'tobedeleted3@gmail.com', + First_Name: 'subcribed3', + Last_Name: ' User3', + }, + type: 'record', + }, + metadata: { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonDeletionDestConfig, + }, + ], + destType, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: 'https://www.zohoapis.in/crm/v6/Leads?ids=&wf_trigger=false', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonDeletionDestConfig, + }, + { + metadata: [ + { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: false, + statusCode: 400, + error: + 'failed to fetch zoho id for record for "No contact is found with record details"', + statTags: { + errorCategory: 'dataValidation', + errorType: 'configuration', + destType: 'ZOHO', + module: 'destination', + implementation: 'cdkV2', + feature: 'router', + }, + destination: commonDeletionDestConfig, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, +]; diff --git a/test/integrations/destinations/zoho/router/upsert.ts b/test/integrations/destinations/zoho/router/upsert.ts new file mode 100644 index 0000000000..a2b898970d --- /dev/null +++ b/test/integrations/destinations/zoho/router/upsert.ts @@ -0,0 +1,771 @@ +import { defaultMockFns } from '../mocks'; +import { + commonOutput1, + commonUpsertDestConfig, + commonUpsertDestConfig2, + commonUpsertDestConfig2CustomModule, + commonUpsertDestConfig3, + destType, + upsertPayload1, + upsertPayload2, + upsertPayload3, +} from '../common'; + +export const upsertData = [ + { + name: destType, + description: 'Happy flow with Leads module', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: upsertPayload1, + metadata: { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig, + }, + { + message: upsertPayload2, + metadata: { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig, + }, + ], + destType, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.zohoapis.com/crm/v6/Leads/upsert', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + params: {}, + body: { + JSON: { + duplicate_check_fields: ['email', 'Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + 'multi-language': ['Bengali'], + }, + ], + $append_values: { + 'multi-language': 'true', + 'multi class': 'false', + }, + trigger: ['workflow'], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonUpsertDestConfig, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, + { + name: destType, + description: 'Happy flow with Trigger None', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: upsertPayload1, + metadata: { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig2, + }, + { + message: upsertPayload2, + metadata: { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig2, + }, + ], + destType, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.zohoapis.com/crm/v6/Leads/upsert', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + params: {}, + body: { + JSON: { + duplicate_check_fields: ['email', 'Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + 'multi-language': ['Bengali'], + }, + ], + $append_values: { + 'multi-language': 'true', + 'multi class': 'false', + }, + trigger: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonUpsertDestConfig2, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, + { + name: destType, + description: 'Happy flow with custom Module', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + action: 'insert', + context: { + externalId: [ + { + type: 'ZOHO-CUSTOM', + identifierType: 'Email', + }, + ], + mappedToDestination: 'true', + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + Name: 'ABC', + }, + type: 'record', + }, + metadata: { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig2CustomModule, + }, + { + message: { + action: 'insert', + context: { + externalId: [ + { + type: 'ZOHO-CUSTOM', + identifierType: 'Email', + }, + ], + mappedToDestination: 'true', + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + 'multi-language': 'Bengali', + Name: 'ABC', + }, + type: 'record', + }, + metadata: { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig2, + }, + ], + destType, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.zohoapis.com/crm/v6/CUSTOM/upsert', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + params: {}, + body: { + JSON: { + duplicate_check_fields: ['Email', 'Name'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + Name: 'ABC', + }, + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + 'multi-language': ['Bengali'], + Name: 'ABC', + }, + ], + $append_values: { + 'multi-language': 'true', + 'multi class': 'false', + }, + trigger: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonUpsertDestConfig2CustomModule, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, + { + name: destType, + description: 'If module specific mandatory field is absent, event will fail', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + action: 'insert', + context: { + externalId: [ + { + type: 'ZOHO-Leads', + identifierType: 'Email', + }, + ], + mappedToDestination: 'true', + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + type: 'record', + }, + metadata: { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig, + }, + { + message: { + action: 'insert', + context: { + externalId: [ + { + type: 'ZOHO-Leads', + identifierType: 'Email', + }, + ], + mappedToDestination: 'true', + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + 'multi-language': 'Bengali', + }, + type: 'record', + }, + metadata: { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig, + }, + ], + destType, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.zohoapis.com/crm/v6/Leads/upsert', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + params: {}, + body: { + JSON: { + duplicate_check_fields: ['Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + ], + $append_values: { + 'multi-language': 'true', + 'multi class': 'false', + }, + trigger: ['workflow'], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + + metadata: [ + { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonUpsertDestConfig, + }, + { + metadata: [ + { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: false, + statusCode: 400, + error: 'Leads object must have the Last_Name property(ies).', + statTags: { + errorCategory: 'dataValidation', + errorType: 'configuration', + destType: 'ZOHO', + module: 'destination', + implementation: 'cdkV2', + feature: 'router', + }, + destination: commonUpsertDestConfig, + }, + ], + }, + }, + }, + }, + { + name: destType, + description: + 'If multiselect key decision is not set from UI, Rudderstack will consider those as normal fields', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: upsertPayload3, + metadata: { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig3, + }, + ], + destType, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.zohoapis.com/crm/v6/Leads/upsert', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + params: {}, + body: { + JSON: commonOutput1, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonUpsertDestConfig3, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, + { + name: destType, + description: 'Test Batching', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: upsertPayload3, + metadata: { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig3, + }, + { + message: upsertPayload3, + metadata: { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig3, + }, + { + message: upsertPayload3, + metadata: { + jobId: 3, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + destination: commonUpsertDestConfig3, + }, + ], + destType, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.zohoapis.com/crm/v6/Leads/upsert', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + params: {}, + body: { + JSON: { + duplicate_check_fields: ['Email'], + data: [ + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + { + Email: 'subscribed@eewrfrd.com', + First_Name: 'subcribed', + Last_Name: ' User', + }, + ], + $append_values: {}, + trigger: ['workflow'], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 1, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + { + jobId: 2, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonUpsertDestConfig3, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.zohoapis.com/crm/v6/Leads/upsert', + headers: { + Authorization: 'Zoho-oauthtoken correct-access-token', + }, + params: {}, + body: { + JSON: commonOutput1, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + jobId: 3, + userId: 'u1', + secret: { + accessToken: 'correct-access-token', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonUpsertDestConfig3, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, +]; From f0918ed05721e0b7faf73929564f04842a47a3ef Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 22 Jul 2024 12:15:53 +0000 Subject: [PATCH 08/28] chore(release): 1.72.0 --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b01182c27..e4e2400f06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,35 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.72.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.71.3...v1.72.0) (2024-07-22) + + +### Features + +* add support for subscribing for RETL flow ([#3195](https://github.com/rudderlabs/rudder-transformer/issues/3195)) ([cc56004](https://github.com/rudderlabs/rudder-transformer/commit/cc560044ceb769da1f0090da4f690933552b6347)) +* braze source event mapping ([#3527](https://github.com/rudderlabs/rudder-transformer/issues/3527)) ([e357141](https://github.com/rudderlabs/rudder-transformer/commit/e357141d22e5296b6d1cda2e763ac24abfcb66e6)) +* introduces new user fields in titkok ads ([#3575](https://github.com/rudderlabs/rudder-transformer/issues/3575)) ([6304abb](https://github.com/rudderlabs/rudder-transformer/commit/6304abb2346f331b78e927b73e7c2ca17e94f4cf)) +* onboard cordial destination ([#3581](https://github.com/rudderlabs/rudder-transformer/issues/3581)) ([fbcdcd6](https://github.com/rudderlabs/rudder-transformer/commit/fbcdcd609888150efa0da33eec60a4cc7b436d06)) +* onboard new custom destination: wunderkind ([#3456](https://github.com/rudderlabs/rudder-transformer/issues/3456)) ([7f49a01](https://github.com/rudderlabs/rudder-transformer/commit/7f49a01b04322a38c5f96199d21097a9210e80fc)) +* onboarding new destination zoho ([#3555](https://github.com/rudderlabs/rudder-transformer/issues/3555)) ([20aa7f3](https://github.com/rudderlabs/rudder-transformer/commit/20aa7f35e13ad89e8a43fbbb743df73b0c103975)), closes [#3566](https://github.com/rudderlabs/rudder-transformer/issues/3566) +* update webhook destination to support all datatypes ([#3541](https://github.com/rudderlabs/rudder-transformer/issues/3541)) ([448f574](https://github.com/rudderlabs/rudder-transformer/commit/448f57484c57d4a55147e9566149c8b714a191c9)) + + +### Bug Fixes + +* add optional chaining to webengage page event ([bf10ca9](https://github.com/rudderlabs/rudder-transformer/commit/bf10ca9bb34caa571d58f9673872e9db3006f8ef)) +* add optional chaining to webengage page event ([#3570](https://github.com/rudderlabs/rudder-transformer/issues/3570)) ([20205d6](https://github.com/rudderlabs/rudder-transformer/commit/20205d66298f5633d3971888f0866db2c38a50e2)) +* add validation for type in google pubsub ([#3578](https://github.com/rudderlabs/rudder-transformer/issues/3578)) ([1bef212](https://github.com/rudderlabs/rudder-transformer/commit/1bef2126a75324598c2af0ecaffcf582f038af11)) +* adding readiness probe annotations for openfaas ([#3529](https://github.com/rudderlabs/rudder-transformer/issues/3529)) ([2eb92e3](https://github.com/rudderlabs/rudder-transformer/commit/2eb92e3332ef0e8b2f83621fe0130fbc1356fa91)) +* formatting ([a7f5c6a](https://github.com/rudderlabs/rudder-transformer/commit/a7f5c6a11b286d7194d369b2d75afcdf23879c03)) +* **gainsight:** replace myAxios utility with handleHttpRequest utility ([c9f934c](https://github.com/rudderlabs/rudder-transformer/commit/c9f934c87eb878e94f713febb36371ca3d240135)) +* **gainsight:** replace myAxios utility with handleHttpRequest utility ([#3241](https://github.com/rudderlabs/rudder-transformer/issues/3241)) ([04be1aa](https://github.com/rudderlabs/rudder-transformer/commit/04be1aaf438f824ddf61fc2f4d13eb7d8a223a9d)) +* job ordering for hs ([#3319](https://github.com/rudderlabs/rudder-transformer/issues/3319)) ([f840d54](https://github.com/rudderlabs/rudder-transformer/commit/f840d54dcbdc011eeb716dce74f2ecb36e99d0e9)) +* update authErrorCategory for 2 step verification issue for google ads ([b7ab0d2](https://github.com/rudderlabs/rudder-transformer/commit/b7ab0d25dbac9c45e7262a506ff0ea3aa9fe03c9)) +* update authErrorCategory for 2 step verification issue for google ads destinations ([#3552](https://github.com/rudderlabs/rudder-transformer/issues/3552)) ([5a0392e](https://github.com/rudderlabs/rudder-transformer/commit/5a0392ee24301486b7973531be28f8178ef03eab)) +* update python transformation fn ([#3491](https://github.com/rudderlabs/rudder-transformer/issues/3491)) ([f363f35](https://github.com/rudderlabs/rudder-transformer/commit/f363f3512f690e0745165f46587efdbe88f48683)) +* zapier event lower case issue ([#3535](https://github.com/rudderlabs/rudder-transformer/issues/3535)) ([277c1f0](https://github.com/rudderlabs/rudder-transformer/commit/277c1f00606b0ec5974e6bf24dae6749a1679069)) + ### [1.71.3](https://github.com/rudderlabs/rudder-transformer/compare/v1.71.2...v1.71.3) (2024-07-15) diff --git a/package-lock.json b/package-lock.json index 12b2155bea..089bb6b110 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.71.3", + "version": "1.72.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.71.3", + "version": "1.72.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 7893676937..8fdce01b7e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.71.3", + "version": "1.72.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From b77d7157f857d1b416c23e545cdc556205c6f2dd Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Tue, 23 Jul 2024 11:59:44 +0530 Subject: [PATCH 09/28] chore: fix changelog --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4e2400f06..97b2add9f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,25 +11,19 @@ All notable changes to this project will be documented in this file. See [standa * braze source event mapping ([#3527](https://github.com/rudderlabs/rudder-transformer/issues/3527)) ([e357141](https://github.com/rudderlabs/rudder-transformer/commit/e357141d22e5296b6d1cda2e763ac24abfcb66e6)) * introduces new user fields in titkok ads ([#3575](https://github.com/rudderlabs/rudder-transformer/issues/3575)) ([6304abb](https://github.com/rudderlabs/rudder-transformer/commit/6304abb2346f331b78e927b73e7c2ca17e94f4cf)) * onboard cordial destination ([#3581](https://github.com/rudderlabs/rudder-transformer/issues/3581)) ([fbcdcd6](https://github.com/rudderlabs/rudder-transformer/commit/fbcdcd609888150efa0da33eec60a4cc7b436d06)) -* onboard new custom destination: wunderkind ([#3456](https://github.com/rudderlabs/rudder-transformer/issues/3456)) ([7f49a01](https://github.com/rudderlabs/rudder-transformer/commit/7f49a01b04322a38c5f96199d21097a9210e80fc)) * onboarding new destination zoho ([#3555](https://github.com/rudderlabs/rudder-transformer/issues/3555)) ([20aa7f3](https://github.com/rudderlabs/rudder-transformer/commit/20aa7f35e13ad89e8a43fbbb743df73b0c103975)), closes [#3566](https://github.com/rudderlabs/rudder-transformer/issues/3566) * update webhook destination to support all datatypes ([#3541](https://github.com/rudderlabs/rudder-transformer/issues/3541)) ([448f574](https://github.com/rudderlabs/rudder-transformer/commit/448f57484c57d4a55147e9566149c8b714a191c9)) ### Bug Fixes -* add optional chaining to webengage page event ([bf10ca9](https://github.com/rudderlabs/rudder-transformer/commit/bf10ca9bb34caa571d58f9673872e9db3006f8ef)) * add optional chaining to webengage page event ([#3570](https://github.com/rudderlabs/rudder-transformer/issues/3570)) ([20205d6](https://github.com/rudderlabs/rudder-transformer/commit/20205d66298f5633d3971888f0866db2c38a50e2)) * add validation for type in google pubsub ([#3578](https://github.com/rudderlabs/rudder-transformer/issues/3578)) ([1bef212](https://github.com/rudderlabs/rudder-transformer/commit/1bef2126a75324598c2af0ecaffcf582f038af11)) * adding readiness probe annotations for openfaas ([#3529](https://github.com/rudderlabs/rudder-transformer/issues/3529)) ([2eb92e3](https://github.com/rudderlabs/rudder-transformer/commit/2eb92e3332ef0e8b2f83621fe0130fbc1356fa91)) -* formatting ([a7f5c6a](https://github.com/rudderlabs/rudder-transformer/commit/a7f5c6a11b286d7194d369b2d75afcdf23879c03)) -* **gainsight:** replace myAxios utility with handleHttpRequest utility ([c9f934c](https://github.com/rudderlabs/rudder-transformer/commit/c9f934c87eb878e94f713febb36371ca3d240135)) * **gainsight:** replace myAxios utility with handleHttpRequest utility ([#3241](https://github.com/rudderlabs/rudder-transformer/issues/3241)) ([04be1aa](https://github.com/rudderlabs/rudder-transformer/commit/04be1aaf438f824ddf61fc2f4d13eb7d8a223a9d)) * job ordering for hs ([#3319](https://github.com/rudderlabs/rudder-transformer/issues/3319)) ([f840d54](https://github.com/rudderlabs/rudder-transformer/commit/f840d54dcbdc011eeb716dce74f2ecb36e99d0e9)) -* update authErrorCategory for 2 step verification issue for google ads ([b7ab0d2](https://github.com/rudderlabs/rudder-transformer/commit/b7ab0d25dbac9c45e7262a506ff0ea3aa9fe03c9)) * update authErrorCategory for 2 step verification issue for google ads destinations ([#3552](https://github.com/rudderlabs/rudder-transformer/issues/3552)) ([5a0392e](https://github.com/rudderlabs/rudder-transformer/commit/5a0392ee24301486b7973531be28f8178ef03eab)) * update python transformation fn ([#3491](https://github.com/rudderlabs/rudder-transformer/issues/3491)) ([f363f35](https://github.com/rudderlabs/rudder-transformer/commit/f363f3512f690e0745165f46587efdbe88f48683)) -* zapier event lower case issue ([#3535](https://github.com/rudderlabs/rudder-transformer/issues/3535)) ([277c1f0](https://github.com/rudderlabs/rudder-transformer/commit/277c1f00606b0ec5974e6bf24dae6749a1679069)) ### [1.71.3](https://github.com/rudderlabs/rudder-transformer/compare/v1.71.2...v1.71.3) (2024-07-15) From 475ebc104c69a52eaa425a9ed564ea9aca1ecd9c Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Tue, 23 Jul 2024 14:36:52 +0530 Subject: [PATCH 10/28] fix: garl get auth err category (#3590) fix: update getAuthErrCategory for garl destination --- .../networkHandler.js | 2 +- .../dataDelivery/business.ts | 8 +-- .../dataDelivery/data.ts | 3 +- .../dataDelivery/oauth.ts | 70 +++++++++++++++++++ .../network.ts | 34 +++++++++ 5 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/oauth.ts diff --git a/src/v0/destinations/google_adwords_remarketing_lists/networkHandler.js b/src/v0/destinations/google_adwords_remarketing_lists/networkHandler.js index 1247eae0b1..98659fdf88 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/networkHandler.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/networkHandler.js @@ -159,7 +159,7 @@ const gaAudienceRespHandler = (destResponse, stageMsg) => { [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), }, response, - getAuthErrCategory(status), + getAuthErrCategory(destResponse), ); }; diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/business.ts b/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/business.ts index a036da149d..ed1741e939 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/business.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/business.ts @@ -4,20 +4,20 @@ import { generateProxyV1Payload, } from '../../../testUtils'; -const commonHeaders = { +export const commonHeaders = { Authorization: 'Bearer dummy-access', 'Content-Type': 'application/json', 'developer-token': 'dummy-dev-token', }; -const commonParams = { +export const commonParams = { destination: 'google_adwords_remarketing_lists', listId: '709078448', customerId: '7693729833', consent: { adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED' }, }; -const validRequestPayload1 = { +export const validRequestPayload1 = { enablePartialFailure: true, operations: [ { @@ -92,7 +92,7 @@ const invalidArgumentRequestPayload = { const metadataArray = [generateGoogleOAuthMetadata(1)]; -const expectedStatTags = { +export const expectedStatTags = { destType: 'GOOGLE_ADWORDS_REMARKETING_LISTS', destinationId: 'default-destinationId', errorCategory: 'network', diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/data.ts b/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/data.ts index 51827a38e2..8bde58c780 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/data.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/data.ts @@ -1,3 +1,4 @@ import { testScenariosForV0API, testScenariosForV1API } from './business'; +import { oauthError } from './oauth'; -export const data = [...testScenariosForV0API, ...testScenariosForV1API]; +export const data = [...testScenariosForV0API, ...testScenariosForV1API, ...oauthError]; diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/oauth.ts b/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/oauth.ts new file mode 100644 index 0000000000..b7f93a6945 --- /dev/null +++ b/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/oauth.ts @@ -0,0 +1,70 @@ +import { generateProxyV1Payload } from '../../../testUtils'; +import { commonHeaders, commonParams, expectedStatTags, validRequestPayload1 } from './business'; + +export const oauthError = [ + { + id: 'garl_oauth_scenario', + name: 'google_adwords_remarketing_lists', + description: + '[Proxy v1 API] :: Oauth where valid credentials are missing as mock response from destination', + successCriteria: 'The proxy should return 401 with authErrorCategory as REFRESH_TOKEN', + scenario: 'Oauth', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload({ + headers: commonHeaders, + params: commonParams, + JSON: validRequestPayload1, + endpoint: 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs', + accessToken: 'dummy-access', + }), + method: 'POST', + }, + }, + output: { + response: { + status: 401, + body: { + output: { + authErrorCategory: 'REFRESH_TOKEN', + message: + 'Request had invalid authentication credentials. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project. during ga_audience response transformation', + response: [ + { + error: + 'Request had invalid authentication credentials. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project. during ga_audience response transformation', + metadata: { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + jobId: 1, + secret: { + accessToken: 'dummy-access', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + }, + statusCode: 401, + }, + ], + statTags: { + destType: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + status: 401, + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/network.ts b/test/integrations/destinations/google_adwords_remarketing_lists/network.ts index 8e7c0acbcf..1fd95858a1 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/network.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/network.ts @@ -207,4 +207,38 @@ export const networkCallsData = [ data: {}, }, }, + { + httpReq: { + url: 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs:create', + data: { + job: { + type: 'CUSTOMER_MATCH_USER_LIST', + customerMatchUserListMetadata: { + userList: 'customers/7693729833/userLists/709078448', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + }, + }, + }, + headers: { + Authorization: 'Bearer dummy-access', + 'Content-Type': 'application/json', + 'developer-token': 'dummy-dev-token', + }, + method: 'POST', + }, + httpRes: { + status: 401, + data: { + error: { + code: 401, + message: + 'Request had invalid authentication credentials. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.', + status: 'UNAUTHENTICATED', + }, + }, + }, + }, ]; From bcabf227256f970e8b981cb1328c0d3e49a1dcf0 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Tue, 23 Jul 2024 11:08:38 +0000 Subject: [PATCH 11/28] chore(release): 1.72.1 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b2add9f0..59bcb63fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.72.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.72.0...v1.72.1) (2024-07-23) + + +### Bug Fixes + +* garl get auth err category ([#3590](https://github.com/rudderlabs/rudder-transformer/issues/3590)) ([475ebc1](https://github.com/rudderlabs/rudder-transformer/commit/475ebc104c69a52eaa425a9ed564ea9aca1ecd9c)) + ## [1.72.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.71.3...v1.72.0) (2024-07-22) diff --git a/package-lock.json b/package-lock.json index 089bb6b110..f68ebefd4c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.72.0", + "version": "1.72.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.72.0", + "version": "1.72.1", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 8fdce01b7e..ea20285752 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.72.0", + "version": "1.72.1", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 96674af4ba006796175bd0c10774b88370cd1ef9 Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Tue, 23 Jul 2024 16:55:33 +0530 Subject: [PATCH 12/28] chore: fix typo --- src/v0/destinations/klaviyo/transform.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js index 4d3d0a8438..eac9dda8a0 100644 --- a/src/v0/destinations/klaviyo/transform.js +++ b/src/v0/destinations/klaviyo/transform.js @@ -20,7 +20,7 @@ const { batchSubscribeEvents, getIdFromNewOrExistingProfile, profileUpdateResponseBuilder, - addSubcribeFlagToTraits, + addSubscribeFlagToTraits, } = require('./util'); const { defaultRequestConfig, @@ -63,7 +63,7 @@ const identifyRequestHandler = async ( if (mappedToDestination) { addExternalIdToTraits(message); adduserIdFromExternalId(message); - traitsInfo = addSubcribeFlagToTraits(traitsInfo); + traitsInfo = addSubscribeFlagToTraits(traitsInfo); } let propertyPayload = constructPayload(message, MAPPING_CONFIG[category.name]); From 8508b4e75655c8c66b9e8c13239ff4ccb50b15e8 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Tue, 23 Jul 2024 11:30:38 +0000 Subject: [PATCH 13/28] chore(release): 1.72.2 --- CHANGELOG.md | 2 ++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59bcb63fd6..39548ce37c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.72.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.72.1...v1.72.2) (2024-07-23) + ### [1.72.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.72.0...v1.72.1) (2024-07-23) diff --git a/package-lock.json b/package-lock.json index f68ebefd4c..c4619cc586 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.72.1", + "version": "1.72.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.72.1", + "version": "1.72.2", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index ea20285752..94cf607c50 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.72.1", + "version": "1.72.2", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 39d30d330bd51d8db07c0bc8936e56dec34c2034 Mon Sep 17 00:00:00 2001 From: Leonidas Vrachnis Date: Tue, 23 Jul 2024 16:30:45 +0200 Subject: [PATCH 14/28] chore: provide go test cases for webhook sources (#3549) --- go/.gitignore | 2 + go/README.md | 23 + go/go.mod | 11 + go/go.sum | 9 + go/webhook/testcases/testcases.go | 99 ++++ go/webhook/testcases/testdata/context.json | 5 + .../testcases/adjust/simple_track_call.json | 66 +++ ...e_track_call_with_no_query_parameters.json | 25 + .../auth0/add_member_to_an_organization.json | 126 ++++ .../testdata/testcases/auth0/empty_batch.json | 21 + .../testcases/auth0/missing_user_id.json | 97 +++ ...er_id_for_all_the_requests_in_a_batch.json | 143 +++++ .../testcases/auth0/successful_signup.json | 506 ++++++++++++++++ .../auth0/update_tenant_settings.json | 557 ++++++++++++++++++ .../testcases/close_crm/group_creation.json | 102 ++++ .../testcases/close_crm/lead_deletion.json | 105 ++++ .../testcases/close_crm/lead_update.json | 147 +++++ .../testcases/moengage/batch_of_events.json | 389 ++++++++++++ .../testcases/moengage/simple_track_call.json | 246 ++++++++ .../testcases/ortto/simple_track_call.json | 108 ++++ ...mple_track_call_with_unknown_field_id.json | 104 ++++ .../revenuecat/initial_purchase_event.json | 121 ++++ .../purchase_event_with_anonymous_user.json | 112 ++++ .../revenuecat/simple_track_call.json | 149 +++++ .../testcases/shopify/carts_create.json | 16 + .../shopify/fullfillments_updated_event.json | 251 ++++++++ .../testcases/shopify/identify_call.json | 174 ++++++ .../testcases/shopify/invalid_topic.json | 24 + .../testcases/shopify/no_query_params.json | 19 + .../shopify/unsupported_checkout_event.json | 50 ++ .../shopify/unsupported_event_type.json | 16 + .../testcases/slack/message_event.json | 145 +++++ .../testdata/testcases/slack/msg_event.json | 149 +++++ .../testdata/testcases/slack/team_joined.json | 89 +++ .../testcases/slack/team_joined_event.json | 85 +++ .../testcases/slack/verification.json | 31 + .../slack/webhook_url_verificatin_event.json | 26 + test/integrations/sources/adjust/data.ts | 3 + test/integrations/sources/auth0/data.ts | 6 + test/integrations/testTypes.ts | 3 + test/integrations/testUtils.ts | 6 +- test/scripts/generateJson.ts | 141 +++++ tsconfig.json | 2 +- 43 files changed, 4506 insertions(+), 3 deletions(-) create mode 100644 go/.gitignore create mode 100644 go/README.md create mode 100644 go/go.mod create mode 100644 go/go.sum create mode 100644 go/webhook/testcases/testcases.go create mode 100644 go/webhook/testcases/testdata/context.json create mode 100644 go/webhook/testcases/testdata/testcases/adjust/simple_track_call.json create mode 100644 go/webhook/testcases/testdata/testcases/adjust/simple_track_call_with_no_query_parameters.json create mode 100644 go/webhook/testcases/testdata/testcases/auth0/add_member_to_an_organization.json create mode 100644 go/webhook/testcases/testdata/testcases/auth0/empty_batch.json create mode 100644 go/webhook/testcases/testdata/testcases/auth0/missing_user_id.json create mode 100644 go/webhook/testcases/testdata/testcases/auth0/missing_user_id_for_all_the_requests_in_a_batch.json create mode 100644 go/webhook/testcases/testdata/testcases/auth0/successful_signup.json create mode 100644 go/webhook/testcases/testdata/testcases/auth0/update_tenant_settings.json create mode 100644 go/webhook/testcases/testdata/testcases/close_crm/group_creation.json create mode 100644 go/webhook/testcases/testdata/testcases/close_crm/lead_deletion.json create mode 100644 go/webhook/testcases/testdata/testcases/close_crm/lead_update.json create mode 100644 go/webhook/testcases/testdata/testcases/moengage/batch_of_events.json create mode 100644 go/webhook/testcases/testdata/testcases/moengage/simple_track_call.json create mode 100644 go/webhook/testcases/testdata/testcases/ortto/simple_track_call.json create mode 100644 go/webhook/testcases/testdata/testcases/ortto/simple_track_call_with_unknown_field_id.json create mode 100644 go/webhook/testcases/testdata/testcases/revenuecat/initial_purchase_event.json create mode 100644 go/webhook/testcases/testdata/testcases/revenuecat/purchase_event_with_anonymous_user.json create mode 100644 go/webhook/testcases/testdata/testcases/revenuecat/simple_track_call.json create mode 100644 go/webhook/testcases/testdata/testcases/shopify/carts_create.json create mode 100644 go/webhook/testcases/testdata/testcases/shopify/fullfillments_updated_event.json create mode 100644 go/webhook/testcases/testdata/testcases/shopify/identify_call.json create mode 100644 go/webhook/testcases/testdata/testcases/shopify/invalid_topic.json create mode 100644 go/webhook/testcases/testdata/testcases/shopify/no_query_params.json create mode 100644 go/webhook/testcases/testdata/testcases/shopify/unsupported_checkout_event.json create mode 100644 go/webhook/testcases/testdata/testcases/shopify/unsupported_event_type.json create mode 100644 go/webhook/testcases/testdata/testcases/slack/message_event.json create mode 100644 go/webhook/testcases/testdata/testcases/slack/msg_event.json create mode 100644 go/webhook/testcases/testdata/testcases/slack/team_joined.json create mode 100644 go/webhook/testcases/testdata/testcases/slack/team_joined_event.json create mode 100644 go/webhook/testcases/testdata/testcases/slack/verification.json create mode 100644 go/webhook/testcases/testdata/testcases/slack/webhook_url_verificatin_event.json create mode 100644 test/scripts/generateJson.ts diff --git a/go/.gitignore b/go/.gitignore new file mode 100644 index 0000000000..d1147c6492 --- /dev/null +++ b/go/.gitignore @@ -0,0 +1,2 @@ +go.work +go.work.sum \ No newline at end of file diff --git a/go/README.md b/go/README.md new file mode 100644 index 0000000000..5286119183 --- /dev/null +++ b/go/README.md @@ -0,0 +1,23 @@ +# GO libraries + +## webhook/testdata + +To generate the test files use: + +```bash +go generate ./... +``` + +Work against local rudder-server: + +```bash +go work init +go work use . +go work use ../../rudder-server +``` + +Then run the webhook tests: + +```bash +go test github.com/rudderlabs/rudder-server/gateway/webhook -count 1 -timeout 2m +``` diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 0000000000..26ebf86140 --- /dev/null +++ b/go/go.mod @@ -0,0 +1,11 @@ +module github.com/rudderlabs/rudder-transformer/go + +go 1.22.4 + +require github.com/stretchr/testify v1.9.0 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go/go.sum b/go/go.sum new file mode 100644 index 0000000000..e20fa14b0b --- /dev/null +++ b/go/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/webhook/testcases/testcases.go b/go/webhook/testcases/testcases.go new file mode 100644 index 0000000000..92a808dcbd --- /dev/null +++ b/go/webhook/testcases/testcases.go @@ -0,0 +1,99 @@ +package testcases + +import ( + "embed" + "encoding/json" + "io/fs" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type Setup struct { + Context Context + Cases []Case +} + +type Context struct { + Now time.Time + RequestIP string `json:"request_ip"` +} + +type Case struct { + Name string + Description string + Skip string + Input Input + Output Output +} + +type Input struct { + Request Request +} +type Request struct { + Method string + RawQuery string `json:"query"` + Headers map[string]string + Body json.RawMessage +} + +type Output struct { + Response Response + Queue []json.RawMessage + ErrQueue []json.RawMessage `json:"err_queue"` +} + +type Response struct { + Body json.RawMessage + StatusCode int `json:"status"` +} + +//go:generate npx ts-node ../../../test/scripts/generateJson.ts sources ./testdata/testcases +//go:generate npx prettier --write ./testdata/**/*.json + +//go:embed testdata/context.json +var contextData []byte + +//go:embed testdata/testcases/**/*.json +var testdata embed.FS + +func Load(t *testing.T) Setup { + t.Helper() + + var tc Context + err := json.Unmarshal(contextData, &tc) + require.NoError(t, err) + + var tcs []Case + err = fs.WalkDir(testdata, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + return nil + } + + f, err := testdata.Open(path) + if err != nil { + return err + } + defer f.Close() + + var tc Case + err = json.NewDecoder(f).Decode(&tc) + if err != nil { + return err + } + tcs = append(tcs, tc) + + return nil + }) + require.NoError(t, err) + + return Setup{ + Context: tc, + Cases: tcs, + } +} diff --git a/go/webhook/testcases/testdata/context.json b/go/webhook/testcases/testdata/context.json new file mode 100644 index 0000000000..1b1f135638 --- /dev/null +++ b/go/webhook/testcases/testdata/context.json @@ -0,0 +1,5 @@ +{ + "request_ip_comment": "192.0.2.x/24 - This block is assigned as \"TEST-NET\" for use in documentation and example code.", + "request_ip": "192.0.2.30", + "now": "2024-03-03T04:48:29.000Z" +} diff --git a/go/webhook/testcases/testdata/testcases/adjust/simple_track_call.json b/go/webhook/testcases/testdata/testcases/adjust/simple_track_call.json new file mode 100644 index 0000000000..ce508cacff --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/adjust/simple_track_call.json @@ -0,0 +1,66 @@ +{ + "name": "adjust", + "description": "Simple track call", + "input": { + "request": { + "body": { + "id": "adjust", + "query_parameters": { + "gps_adid": ["38400000-8cf0-11bd-b23e-10b96e40000d"], + "adid": ["18546f6171f67e29d1cb983322ad1329"], + "tracker_token": ["abc"], + "custom": ["custom"], + "tracker_name": ["dummy"], + "created_at": ["1404214665"], + "event_name": ["Click"] + }, + "updated_at": "2023-02-10T12:16:07.251Z", + "created_at": "2023-02-10T12:05:04.402Z" + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "Adjust" + }, + "device": { + "id ": "18546f6171f67e29d1cb983322ad1329" + } + }, + "integrations": { + "Adjust": false + }, + "type": "track", + "event": "Click", + "originalTimestamp": "2014-07-01T11:37:45.000Z", + "timestamp": "2014-07-01T11:37:45.000Z", + "properties": { + "gps_adid": "38400000-8cf0-11bd-b23e-10b96e40000d", + "tracker_token": "abc", + "custom": "custom", + "tracker_name": "dummy" + }, + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ], + "errQueue": [] + }, + "skip": "FIXME" +} diff --git a/go/webhook/testcases/testdata/testcases/adjust/simple_track_call_with_no_query_parameters.json b/go/webhook/testcases/testdata/testcases/adjust/simple_track_call_with_no_query_parameters.json new file mode 100644 index 0000000000..f8e5a480a0 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/adjust/simple_track_call_with_no_query_parameters.json @@ -0,0 +1,25 @@ +{ + "name": "adjust", + "description": "Simple track call with no query parameters", + "input": { + "request": { + "body": { + "id": "adjust", + "updated_at": "2023-02-10T12:16:07.251Z", + "created_at": "2023-02-10T12:05:04.402Z" + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 400, + "body": "Query_parameters is missing" + }, + "queue": [], + "errQueue": [null] + }, + "skip": "FIXME" +} diff --git a/go/webhook/testcases/testdata/testcases/auth0/add_member_to_an_organization.json b/go/webhook/testcases/testdata/testcases/auth0/add_member_to_an_organization.json new file mode 100644 index 0000000000..77fbd643c8 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/auth0/add_member_to_an_organization.json @@ -0,0 +1,126 @@ +{ + "name": "auth0", + "description": "add member to an organization", + "input": { + "request": { + "body": { + "log_id": "90020221031061004280169676882609459981150114445973782546", + "data": { + "date": "2022-10-31T06:09:59.135Z", + "type": "sapi", + "description": "Add members to an organization", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "", + "ip": "35.167.74.121", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "details": { + "request": { + "ip": "35.167.74.121", + "auth": { + "user": { + "name": "rudder test", + "email": "test@rudderstack.com", + "user_id": "google-oauth2|123456" + }, + "strategy": "jwt", + "credentials": { + "jti": "571921bf7833a97efabf08d765a0ec8f" + } + }, + "body": { + "members": ["auth0|123456"] + }, + "path": "/api/v2/organizations/org_eoe8p2atZ7furBxg/members", + "query": {}, + "method": "post", + "channel": "https://manage.auth0.com/", + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + }, + "response": { + "body": {}, + "statusCode": 204 + } + }, + "user_id": "google-oauth2|123456", + "log_id": "90020221031061004280169676882609459981150114445973782546" + } + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "type": "group", + "sentAt": "2022-10-31T06:09:59.135Z", + "userId": "google-oauth2|123456", + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "traits": { + "userId": "google-oauth2|123456" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "request_ip": "35.167.74.121", + "integration": { + "name": "Auth0" + } + }, + "groupId": "org_eoe8p2atZ7furBxg", + "properties": { + "log_id": "90020221031061004280169676882609459981150114445973782546", + "details": { + "request": { + "ip": "35.167.74.121", + "auth": { + "user": { + "name": "rudder test", + "email": "test@rudderstack.com", + "user_id": "google-oauth2|123456" + }, + "strategy": "jwt", + "credentials": { + "jti": "571921bf7833a97efabf08d765a0ec8f" + } + }, + "body": { + "members": ["auth0|123456"] + }, + "path": "/api/v2/organizations/org_eoe8p2atZ7furBxg/members", + "query": {}, + "method": "post", + "channel": "https://manage.auth0.com/", + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + }, + "response": { + "body": {}, + "statusCode": 204 + } + }, + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "", + "description": "Add members to an organization", + "source_type": "sapi" + }, + "integrations": { + "Auth0": false + }, + "originalTimestamp": "2022-10-31T06:09:59.135Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ], + "errQueue": [] + }, + "skip": "dynamic anonymousId" +} diff --git a/go/webhook/testcases/testdata/testcases/auth0/empty_batch.json b/go/webhook/testcases/testdata/testcases/auth0/empty_batch.json new file mode 100644 index 0000000000..709ee35525 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/auth0/empty_batch.json @@ -0,0 +1,21 @@ +{ + "name": "auth0", + "description": "empty batch", + "input": { + "request": { + "body": [], + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [], + "errQueue": [] + }, + "skip": "dynamic anonymousId" +} diff --git a/go/webhook/testcases/testdata/testcases/auth0/missing_user_id.json b/go/webhook/testcases/testdata/testcases/auth0/missing_user_id.json new file mode 100644 index 0000000000..8dc148b4c2 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/auth0/missing_user_id.json @@ -0,0 +1,97 @@ +{ + "name": "auth0", + "description": "missing userId", + "input": { + "request": { + "body": { + "log_id": "90020221031055712103169676686005480714681762668315934738", + "data": { + "date": "2022-10-31T05:57:06.859Z", + "type": "ss", + "description": "", + "connection": "Username-Password-Authentication", + "connection_id": "con_djwCjiwyID0vZy1S", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "All Applications", + "ip": "35.166.202.113", + "user_agent": "unknown", + "details": { + "body": { + "email": "testRudderlabs+21@gmail.com", + "tenant": "dev-cu4jy2zgao6yx15x", + "password": "dummyPassword", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "connection": "Username-Password-Authentication" + } + }, + "user_id": "", + "user_name": "testRudderlabs+21@gmail.com", + "strategy": "auth0", + "strategy_type": "database", + "log_id": "90020221031055712103169676686005480714681762668315934738" + } + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "type": "identify", + "sentAt": "2022-10-31T05:57:06.859Z", + "traits": { + "connection": "Username-Password-Authentication", + "connection_id": "con_djwCjiwyID0vZy1S" + }, + "userId": "", + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "context": { + "traits": { + "userId": "", + "user_name": "testRudderlabs+21@gmail.com" + }, + "library": { + "name": "unknown", + "version": "unknown" + }, + "userAgent": "unknown", + "request_ip": "35.166.202.113", + "integration": { + "name": "Auth0" + } + }, + "properties": { + "log_id": "90020221031055712103169676686005480714681762668315934738", + "details": { + "body": { + "email": "testRudderlabs+21@gmail.com", + "tenant": "dev-cu4jy2zgao6yx15x", + "password": "dummyPassword", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "connection": "Username-Password-Authentication" + } + }, + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "All Applications", + "description": "", + "source_type": "ss" + }, + "integrations": { + "Auth0": false + }, + "originalTimestamp": "2022-10-31T05:57:06.859Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ], + "errQueue": [] + }, + "skip": "dynamic anonymousId" +} diff --git a/go/webhook/testcases/testdata/testcases/auth0/missing_user_id_for_all_the_requests_in_a_batch.json b/go/webhook/testcases/testdata/testcases/auth0/missing_user_id_for_all_the_requests_in_a_batch.json new file mode 100644 index 0000000000..774ab7e7e2 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/auth0/missing_user_id_for_all_the_requests_in_a_batch.json @@ -0,0 +1,143 @@ +{ + "name": "auth0", + "description": "missing userId for all the requests in a batch", + "input": { + "request": { + "body": [ + { + "log_id": "90020221031055712103169676686005480714681762668315934738", + "data": { + "date": "2022-10-31T05:57:06.859Z", + "type": "ss", + "description": "", + "connection": "Username-Password-Authentication", + "connection_id": "con_djwCjiwyID0vZy1S", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "All Applications", + "ip": "35.166.202.113", + "user_agent": "unknown", + "details": { + "body": { + "email": "testRudderlabs+21@gmail.com", + "tenant": "dev-cu4jy2zgao6yx15x", + "password": "dummyPassword", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "connection": "Username-Password-Authentication" + } + }, + "user_id": "", + "user_name": "testRudderlabs+21@gmail.com", + "strategy": "auth0", + "strategy_type": "database", + "log_id": "90020221031055712103169676686005480714681762668315934738" + } + }, + { + "log_id": "90020221031055712103169676686007898566320991926665347090", + "data": { + "date": "2022-10-31T05:57:06.874Z", + "type": "sapi", + "description": "Create a User", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "", + "ip": "35.166.202.113", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "log_id": "90020221031055712103169676686007898566320991926665347090" + } + } + ], + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "type": "identify", + "userId": "", + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "sentAt": "2022-10-31T05:57:06.859Z", + "traits": { + "connection": "Username-Password-Authentication", + "connection_id": "con_djwCjiwyID0vZy1S" + }, + "context": { + "traits": { + "userId": "", + "user_name": "testRudderlabs+21@gmail.com" + }, + "library": { + "name": "unknown", + "version": "unknown" + }, + "userAgent": "unknown", + "request_ip": "35.166.202.113", + "integration": { + "name": "Auth0" + } + }, + "properties": { + "log_id": "90020221031055712103169676686005480714681762668315934738", + "details": { + "body": { + "email": "testRudderlabs+21@gmail.com", + "tenant": "dev-cu4jy2zgao6yx15x", + "password": "dummyPassword", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "connection": "Username-Password-Authentication" + } + }, + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "All Applications", + "description": "", + "source_type": "ss" + }, + "integrations": { + "Auth0": false + }, + "originalTimestamp": "2022-10-31T05:57:06.859Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + }, + { + "type": "track", + "event": "Success API Operation", + "sentAt": "2022-10-31T05:57:06.874Z", + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "request_ip": "35.166.202.113", + "integration": { + "name": "Auth0" + } + }, + "properties": { + "log_id": "90020221031055712103169676686007898566320991926665347090", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "", + "description": "Create a User", + "source_type": "sapi" + }, + "integrations": { + "Auth0": false + }, + "originalTimestamp": "2022-10-31T05:57:06.874Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ], + "errQueue": [] + }, + "skip": "dynamic anonymousId" +} diff --git a/go/webhook/testcases/testdata/testcases/auth0/successful_signup.json b/go/webhook/testcases/testdata/testcases/auth0/successful_signup.json new file mode 100644 index 0000000000..91fba2bd4b --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/auth0/successful_signup.json @@ -0,0 +1,506 @@ +{ + "name": "auth0", + "description": "successful signup", + "input": { + "request": { + "body": [ + { + "log_id": "90020221031055712103169676686005480714681762668315934738", + "data": { + "date": "2022-10-31T05:57:06.859Z", + "type": "ss", + "description": "", + "connection": "Username-Password-Authentication", + "connection_id": "con_djwCjiwyID0vZy1S", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "All Applications", + "ip": "35.166.202.113", + "user_agent": "unknown", + "details": { + "body": { + "email": "testRudderlabs+21@gmail.com", + "tenant": "dev-cu4jy2zgao6yx15x", + "password": "dummyPassword", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "connection": "Username-Password-Authentication" + } + }, + "user_id": "auth0|dummyPassword", + "user_name": "testRudderlabs+21@gmail.com", + "strategy": "auth0", + "strategy_type": "database", + "log_id": "90020221031055712103169676686005480714681762668315934738" + } + }, + { + "log_id": "90020221031055712103169676686007898566320991926665347090", + "data": { + "date": "2022-10-31T05:57:06.874Z", + "type": "sapi", + "description": "Create a User", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "", + "ip": "35.166.202.113", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "details": { + "request": { + "ip": "35.166.202.113", + "auth": { + "user": { + "name": "rudder test", + "email": "test@rudderstack.com", + "user_id": "auth0|dummyPassword" + }, + "strategy": "jwt", + "credentials": { + "jti": "571921bf7833a97efabf08d765a0ec8f", + "scopes": [ + "create:actions", + "create:actions_log_sessions", + "create:client_credentials", + "create:client_grants", + "create:clients", + "create:connections", + "create:custom_domains", + "create:email_provider", + "create:email_templates", + "create:guardian_enrollment_tickets", + "create:integrations", + "create:log_streams", + "create:organization_connections", + "create:organization_invitations", + "create:organization_member_roles", + "create:organization_members", + "create:organizations", + "create:requested_scopes", + "create:resource_servers", + "create:roles", + "create:rules", + "create:shields", + "create:signing_keys", + "create:tenant_invitations", + "create:test_email_dispatch", + "create:users", + "delete:actions", + "delete:anomaly_blocks", + "delete:branding", + "delete:client_credentials", + "delete:client_grants", + "delete:clients", + "delete:connections", + "delete:custom_domains", + "delete:device_credentials", + "delete:email_provider", + "delete:email_templates", + "delete:grants", + "delete:guardian_enrollments", + "delete:integrations", + "delete:log_streams", + "delete:organization_connections", + "delete:organization_invitations", + "delete:organization_member_roles", + "delete:organization_members", + "delete:organizations", + "delete:owners", + "delete:requested_scopes", + "delete:resource_servers", + "delete:roles", + "delete:rules", + "delete:rules_configs", + "delete:shields", + "delete:tenant_invitations", + "delete:tenant_members", + "delete:tenants", + "delete:users", + "read:actions", + "read:anomaly_blocks", + "read:attack_protection", + "read:branding", + "read:checks", + "read:client_credentials", + "read:client_grants", + "read:client_keys", + "read:clients", + "read:connections", + "read:custom_domains", + "read:device_credentials", + "read:email_provider", + "read:email_templates", + "read:email_triggers", + "read:entity_counts", + "read:grants", + "read:guardian_factors", + "read:insights", + "read:integrations", + "read:log_streams", + "read:logs", + "read:mfa_policies", + "read:organization_connections", + "read:organization_invitations", + "read:organization_member_roles", + "read:organization_members", + "read:organizations", + "read:prompts", + "read:requested_scopes", + "read:resource_servers", + "read:roles", + "read:rules", + "read:rules_configs", + "read:shields", + "read:signing_keys", + "read:stats", + "read:tenant_invitations", + "read:tenant_members", + "read:tenant_settings", + "read:triggers", + "read:users", + "run:checks", + "update:actions", + "update:attack_protection", + "update:branding", + "update:client_credentials", + "update:client_grants", + "update:client_keys", + "update:clients", + "update:connections", + "update:custom_domains", + "update:email_provider", + "update:email_templates", + "update:email_triggers", + "update:guardian_factors", + "update:integrations", + "update:log_streams", + "update:mfa_policies", + "update:organization_connections", + "update:organizations", + "update:prompts", + "update:requested_scopes", + "update:resource_servers", + "update:roles", + "update:rules", + "update:rules_configs", + "update:shields", + "update:signing_keys", + "update:tenant_members", + "update:tenant_settings", + "update:triggers", + "update:users" + ] + } + }, + "body": { + "email": "testRudderlabs+21@gmail.com", + "password": "dummyPassword", + "connection": "Username-Password-Authentication" + }, + "path": "/api/v2/users", + "query": {}, + "method": "post", + "channel": "https://manage.auth0.com/", + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + }, + "response": { + "body": { + "name": "testRudderlabs+21@gmail.com", + "email": "testRudderlabs+21@gmail.com", + "picture": "https://s.gravatar.com/avatar/0902f9d02b92aed9f0ac59aaf9475b60?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fbh.png", + "user_id": "auth0|dummyPassword", + "nickname": "testRudderlabs+21", + "created_at": "2022-10-31T05:57:06.864Z", + "identities": [ + { + "user_id": "auth0|dummyPassword", + "isSocial": false, + "provider": "auth0", + "connection": "Username-Password-Authentication" + } + ], + "updated_at": "2022-10-31T05:57:06.864Z", + "email_verified": false + }, + "statusCode": 201 + } + }, + "user_id": "auth0|dummyPassword", + "log_id": "90020221031055712103169676686007898566320991926665347090" + } + } + ], + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "type": "identify", + "sentAt": "2022-10-31T05:57:06.859Z", + "traits": { + "connection": "Username-Password-Authentication", + "connection_id": "con_djwCjiwyID0vZy1S" + }, + "userId": "auth0|dummyPassword", + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "context": { + "traits": { + "userId": "auth0|dummyPassword", + "user_name": "testRudderlabs+21@gmail.com" + }, + "library": { + "name": "unknown", + "version": "unknown" + }, + "userAgent": "unknown", + "request_ip": "35.166.202.113", + "integration": { + "name": "Auth0" + } + }, + "properties": { + "log_id": "90020221031055712103169676686005480714681762668315934738", + "details": { + "body": { + "email": "testRudderlabs+21@gmail.com", + "tenant": "dev-cu4jy2zgao6yx15x", + "password": "dummyPassword", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "connection": "Username-Password-Authentication" + } + }, + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "All Applications", + "description": "", + "source_type": "ss" + }, + "integrations": { + "Auth0": false + }, + "originalTimestamp": "2022-10-31T05:57:06.859Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + }, + { + "type": "track", + "event": "Success API Operation", + "sentAt": "2022-10-31T05:57:06.874Z", + "userId": "auth0|dummyPassword", + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "traits": { + "userId": "auth0|dummyPassword" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "request_ip": "35.166.202.113", + "integration": { + "name": "Auth0" + } + }, + "properties": { + "log_id": "90020221031055712103169676686007898566320991926665347090", + "details": { + "request": { + "ip": "35.166.202.113", + "auth": { + "user": { + "name": "rudder test", + "email": "test@rudderstack.com", + "user_id": "auth0|dummyPassword" + }, + "strategy": "jwt", + "credentials": { + "jti": "571921bf7833a97efabf08d765a0ec8f", + "scopes": [ + "create:actions", + "create:actions_log_sessions", + "create:client_credentials", + "create:client_grants", + "create:clients", + "create:connections", + "create:custom_domains", + "create:email_provider", + "create:email_templates", + "create:guardian_enrollment_tickets", + "create:integrations", + "create:log_streams", + "create:organization_connections", + "create:organization_invitations", + "create:organization_member_roles", + "create:organization_members", + "create:organizations", + "create:requested_scopes", + "create:resource_servers", + "create:roles", + "create:rules", + "create:shields", + "create:signing_keys", + "create:tenant_invitations", + "create:test_email_dispatch", + "create:users", + "delete:actions", + "delete:anomaly_blocks", + "delete:branding", + "delete:client_credentials", + "delete:client_grants", + "delete:clients", + "delete:connections", + "delete:custom_domains", + "delete:device_credentials", + "delete:email_provider", + "delete:email_templates", + "delete:grants", + "delete:guardian_enrollments", + "delete:integrations", + "delete:log_streams", + "delete:organization_connections", + "delete:organization_invitations", + "delete:organization_member_roles", + "delete:organization_members", + "delete:organizations", + "delete:owners", + "delete:requested_scopes", + "delete:resource_servers", + "delete:roles", + "delete:rules", + "delete:rules_configs", + "delete:shields", + "delete:tenant_invitations", + "delete:tenant_members", + "delete:tenants", + "delete:users", + "read:actions", + "read:anomaly_blocks", + "read:attack_protection", + "read:branding", + "read:checks", + "read:client_credentials", + "read:client_grants", + "read:client_keys", + "read:clients", + "read:connections", + "read:custom_domains", + "read:device_credentials", + "read:email_provider", + "read:email_templates", + "read:email_triggers", + "read:entity_counts", + "read:grants", + "read:guardian_factors", + "read:insights", + "read:integrations", + "read:log_streams", + "read:logs", + "read:mfa_policies", + "read:organization_connections", + "read:organization_invitations", + "read:organization_member_roles", + "read:organization_members", + "read:organizations", + "read:prompts", + "read:requested_scopes", + "read:resource_servers", + "read:roles", + "read:rules", + "read:rules_configs", + "read:shields", + "read:signing_keys", + "read:stats", + "read:tenant_invitations", + "read:tenant_members", + "read:tenant_settings", + "read:triggers", + "read:users", + "run:checks", + "update:actions", + "update:attack_protection", + "update:branding", + "update:client_credentials", + "update:client_grants", + "update:client_keys", + "update:clients", + "update:connections", + "update:custom_domains", + "update:email_provider", + "update:email_templates", + "update:email_triggers", + "update:guardian_factors", + "update:integrations", + "update:log_streams", + "update:mfa_policies", + "update:organization_connections", + "update:organizations", + "update:prompts", + "update:requested_scopes", + "update:resource_servers", + "update:roles", + "update:rules", + "update:rules_configs", + "update:shields", + "update:signing_keys", + "update:tenant_members", + "update:tenant_settings", + "update:triggers", + "update:users" + ] + } + }, + "body": { + "email": "testRudderlabs+21@gmail.com", + "password": "dummyPassword", + "connection": "Username-Password-Authentication" + }, + "path": "/api/v2/users", + "query": {}, + "method": "post", + "channel": "https://manage.auth0.com/", + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + }, + "response": { + "body": { + "name": "testRudderlabs+21@gmail.com", + "email": "testRudderlabs+21@gmail.com", + "picture": "https://s.gravatar.com/avatar/0902f9d02b92aed9f0ac59aaf9475b60?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fbh.png", + "user_id": "auth0|dummyPassword", + "nickname": "testRudderlabs+21", + "created_at": "2022-10-31T05:57:06.864Z", + "identities": [ + { + "user_id": "auth0|dummyPassword", + "isSocial": false, + "provider": "auth0", + "connection": "Username-Password-Authentication" + } + ], + "updated_at": "2022-10-31T05:57:06.864Z", + "email_verified": false + }, + "statusCode": 201 + } + }, + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "", + "description": "Create a User", + "source_type": "sapi" + }, + "integrations": { + "Auth0": false + }, + "originalTimestamp": "2022-10-31T05:57:06.874Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ], + "errQueue": [] + }, + "skip": "dynamic anonymousId" +} diff --git a/go/webhook/testcases/testdata/testcases/auth0/update_tenant_settings.json b/go/webhook/testcases/testdata/testcases/auth0/update_tenant_settings.json new file mode 100644 index 0000000000..da005184ad --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/auth0/update_tenant_settings.json @@ -0,0 +1,557 @@ +{ + "name": "auth0", + "description": "update tenant settings", + "input": { + "request": { + "body": [ + { + "log_id": "90020221031061527239169676960191065529099349299958906898", + "data": { + "date": "2022-10-31T06:15:25.201Z", + "type": "sapi", + "description": "Update tenant settings", + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "", + "ip": "35.160.3.103", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "details": { + "request": { + "ip": "35.160.3.103", + "auth": { + "user": { + "name": "rudder test", + "email": "test@rudderstack.com", + "user_id": "google-oauth2|123456" + }, + "strategy": "jwt", + "credentials": { + "jti": "571921bf7833a97efabf08d765a0ec8f", + "scopes": [ + "create:actions", + "create:actions_log_sessions", + "create:client_credentials", + "create:client_grants", + "create:clients", + "create:connections", + "create:custom_domains", + "create:email_provider", + "create:email_templates", + "create:guardian_enrollment_tickets", + "create:integrations", + "create:log_streams", + "create:organization_connections", + "create:organization_invitations", + "create:organization_member_roles", + "create:organization_members", + "create:organizations", + "create:requested_scopes", + "create:resource_servers", + "create:roles", + "create:rules", + "create:shields", + "create:signing_keys", + "create:tenant_invitations", + "create:test_email_dispatch", + "create:users", + "delete:actions", + "delete:anomaly_blocks", + "delete:branding", + "delete:client_credentials", + "delete:client_grants", + "delete:clients", + "delete:connections", + "delete:custom_domains", + "delete:device_credentials", + "delete:email_provider", + "delete:email_templates", + "delete:grants", + "delete:guardian_enrollments", + "delete:integrations", + "delete:log_streams", + "delete:organization_connections", + "delete:organization_invitations", + "delete:organization_member_roles", + "delete:organization_members", + "delete:organizations", + "delete:owners", + "delete:requested_scopes", + "delete:resource_servers", + "delete:roles", + "delete:rules", + "delete:rules_configs", + "delete:shields", + "delete:tenant_invitations", + "delete:tenant_members", + "delete:tenants", + "delete:users", + "read:actions", + "read:anomaly_blocks", + "read:attack_protection", + "read:branding", + "read:checks", + "read:client_credentials", + "read:client_grants", + "read:client_keys", + "read:clients", + "read:connections", + "read:custom_domains", + "read:device_credentials", + "read:email_provider", + "read:email_templates", + "read:email_triggers", + "read:entity_counts", + "read:grants", + "read:guardian_factors", + "read:insights", + "read:integrations", + "read:log_streams", + "read:logs", + "read:mfa_policies", + "read:organization_connections", + "read:organization_invitations", + "read:organization_member_roles", + "read:organization_members", + "read:organizations", + "read:prompts", + "read:requested_scopes", + "read:resource_servers", + "read:roles", + "read:rules", + "read:rules_configs", + "read:shields", + "read:signing_keys", + "read:stats", + "read:tenant_invitations", + "read:tenant_members", + "read:tenant_settings", + "read:triggers", + "read:users", + "run:checks", + "update:actions", + "update:attack_protection", + "update:branding", + "update:client_credentials", + "update:client_grants", + "update:client_keys", + "update:clients", + "update:connections", + "update:custom_domains", + "update:email_provider", + "update:email_templates", + "update:email_triggers", + "update:guardian_factors", + "update:integrations", + "update:log_streams", + "update:mfa_policies", + "update:organization_connections", + "update:organizations", + "update:prompts", + "update:requested_scopes", + "update:resource_servers", + "update:roles", + "update:rules", + "update:rules_configs", + "update:shields", + "update:signing_keys", + "update:tenant_members", + "update:tenant_settings", + "update:triggers", + "update:users" + ] + } + }, + "body": { + "picture_url": "", + "support_url": "", + "friendly_name": "mecro-action", + "support_email": "support@test.com" + }, + "path": "/api/v2/tenants/settings", + "query": {}, + "method": "patch", + "channel": "https://manage.auth0.com/", + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + }, + "response": { + "body": { + "flags": { + "enable_sso": true, + "universal_login": true, + "disable_impersonation": true, + "allow_changing_enable_sso": false, + "revoke_refresh_token_grant": false, + "disable_clickjack_protection_headers": false, + "new_universal_login_experience_enabled": true, + "enforce_client_authentication_on_passwordless_start": true, + "cannot_change_enforce_client_authentication_on_passwordless_start": true + }, + "picture_url": "", + "support_url": "", + "friendly_name": "mecro-action", + "support_email": "support@test.com", + "enabled_locales": ["en"], + "sandbox_version": "16", + "universal_login": {} + }, + "statusCode": 200 + } + }, + "user_id": "google-oauth2|123456", + "log_id": "90020221031061527239169676960191065529099349299958906898" + } + }, + { + "log_id": "90020221031061530247169676961198100736838335677367058450", + "data": { + "date": "2022-10-31T06:15:25.196Z", + "type": "gd_tenant_update", + "description": "Guardian - Updates tenant settings", + "ip": "35.160.3.103", + "details": { + "request": { + "ip": "35.160.3.103", + "auth": { + "scopes": [ + "read:authenticators", + "remove:authenticators", + "update:authenticators", + "create:authenticators", + "read:enrollments", + "delete:enrollments", + "read:factors", + "update:factors", + "update:tenant_settings", + "update:users", + "create:enrollment_tickets", + "create:users" + ], + "subject": "google-oauth2|123456", + "strategy": "jwt_api2_internal_token" + }, + "body": { + "picture_url": "[REDACTED]", + "friendly_name": "[REDACTED]" + }, + "path": "/api/tenants/settings", + "query": {}, + "method": "PATCH" + }, + "response": { + "body": { + "name": "dev-cu4jy2zgao6yx15x", + "picture_url": "[REDACTED]", + "friendly_name": "[REDACTED]", + "guardian_mfa_page": "[REDACTED]" + }, + "statusCode": 200 + } + }, + "user_id": "google-oauth2|123456", + "log_id": "90020221031061530247169676961198100736838335677367058450" + } + } + ], + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "type": "track", + "event": "Success API Operation", + "sentAt": "2022-10-31T06:15:25.201Z", + "userId": "google-oauth2|123456", + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "traits": { + "userId": "google-oauth2|123456" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "request_ip": "35.160.3.103", + "integration": { + "name": "Auth0" + } + }, + "properties": { + "log_id": "90020221031061527239169676960191065529099349299958906898", + "details": { + "request": { + "ip": "35.160.3.103", + "auth": { + "user": { + "name": "rudder test", + "email": "test@rudderstack.com", + "user_id": "google-oauth2|123456" + }, + "strategy": "jwt", + "credentials": { + "jti": "571921bf7833a97efabf08d765a0ec8f", + "scopes": [ + "create:actions", + "create:actions_log_sessions", + "create:client_credentials", + "create:client_grants", + "create:clients", + "create:connections", + "create:custom_domains", + "create:email_provider", + "create:email_templates", + "create:guardian_enrollment_tickets", + "create:integrations", + "create:log_streams", + "create:organization_connections", + "create:organization_invitations", + "create:organization_member_roles", + "create:organization_members", + "create:organizations", + "create:requested_scopes", + "create:resource_servers", + "create:roles", + "create:rules", + "create:shields", + "create:signing_keys", + "create:tenant_invitations", + "create:test_email_dispatch", + "create:users", + "delete:actions", + "delete:anomaly_blocks", + "delete:branding", + "delete:client_credentials", + "delete:client_grants", + "delete:clients", + "delete:connections", + "delete:custom_domains", + "delete:device_credentials", + "delete:email_provider", + "delete:email_templates", + "delete:grants", + "delete:guardian_enrollments", + "delete:integrations", + "delete:log_streams", + "delete:organization_connections", + "delete:organization_invitations", + "delete:organization_member_roles", + "delete:organization_members", + "delete:organizations", + "delete:owners", + "delete:requested_scopes", + "delete:resource_servers", + "delete:roles", + "delete:rules", + "delete:rules_configs", + "delete:shields", + "delete:tenant_invitations", + "delete:tenant_members", + "delete:tenants", + "delete:users", + "read:actions", + "read:anomaly_blocks", + "read:attack_protection", + "read:branding", + "read:checks", + "read:client_credentials", + "read:client_grants", + "read:client_keys", + "read:clients", + "read:connections", + "read:custom_domains", + "read:device_credentials", + "read:email_provider", + "read:email_templates", + "read:email_triggers", + "read:entity_counts", + "read:grants", + "read:guardian_factors", + "read:insights", + "read:integrations", + "read:log_streams", + "read:logs", + "read:mfa_policies", + "read:organization_connections", + "read:organization_invitations", + "read:organization_member_roles", + "read:organization_members", + "read:organizations", + "read:prompts", + "read:requested_scopes", + "read:resource_servers", + "read:roles", + "read:rules", + "read:rules_configs", + "read:shields", + "read:signing_keys", + "read:stats", + "read:tenant_invitations", + "read:tenant_members", + "read:tenant_settings", + "read:triggers", + "read:users", + "run:checks", + "update:actions", + "update:attack_protection", + "update:branding", + "update:client_credentials", + "update:client_grants", + "update:client_keys", + "update:clients", + "update:connections", + "update:custom_domains", + "update:email_provider", + "update:email_templates", + "update:email_triggers", + "update:guardian_factors", + "update:integrations", + "update:log_streams", + "update:mfa_policies", + "update:organization_connections", + "update:organizations", + "update:prompts", + "update:requested_scopes", + "update:resource_servers", + "update:roles", + "update:rules", + "update:rules_configs", + "update:shields", + "update:signing_keys", + "update:tenant_members", + "update:tenant_settings", + "update:triggers", + "update:users" + ] + } + }, + "body": { + "picture_url": "", + "support_url": "", + "friendly_name": "mecro-action", + "support_email": "support@test.com" + }, + "path": "/api/v2/tenants/settings", + "query": {}, + "method": "patch", + "channel": "https://manage.auth0.com/", + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + }, + "response": { + "body": { + "flags": { + "enable_sso": true, + "universal_login": true, + "disable_impersonation": true, + "allow_changing_enable_sso": false, + "revoke_refresh_token_grant": false, + "disable_clickjack_protection_headers": false, + "new_universal_login_experience_enabled": true, + "enforce_client_authentication_on_passwordless_start": true, + "cannot_change_enforce_client_authentication_on_passwordless_start": true + }, + "picture_url": "", + "support_url": "", + "friendly_name": "mecro-action", + "support_email": "support@test.com", + "enabled_locales": ["en"], + "sandbox_version": "16", + "universal_login": {} + }, + "statusCode": 200 + } + }, + "client_id": "vQcJNDTxsM1W72eHFonRJdzyOvawlwIt", + "client_name": "", + "description": "Update tenant settings", + "source_type": "sapi" + }, + "integrations": { + "Auth0": false + }, + "originalTimestamp": "2022-10-31T06:15:25.201Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + }, + { + "type": "track", + "event": "Guardian tenant update", + "sentAt": "2022-10-31T06:15:25.196Z", + "userId": "google-oauth2|123456", + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "traits": { + "userId": "google-oauth2|123456" + }, + "request_ip": "35.160.3.103", + "integration": { + "name": "Auth0" + } + }, + "properties": { + "log_id": "90020221031061530247169676961198100736838335677367058450", + "details": { + "request": { + "ip": "35.160.3.103", + "auth": { + "scopes": [ + "read:authenticators", + "remove:authenticators", + "update:authenticators", + "create:authenticators", + "read:enrollments", + "delete:enrollments", + "read:factors", + "update:factors", + "update:tenant_settings", + "update:users", + "create:enrollment_tickets", + "create:users" + ], + "subject": "google-oauth2|123456", + "strategy": "jwt_api2_internal_token" + }, + "body": { + "picture_url": "[REDACTED]", + "friendly_name": "[REDACTED]" + }, + "path": "/api/tenants/settings", + "query": {}, + "method": "PATCH" + }, + "response": { + "body": { + "name": "dev-cu4jy2zgao6yx15x", + "picture_url": "[REDACTED]", + "friendly_name": "[REDACTED]", + "guardian_mfa_page": "[REDACTED]" + }, + "statusCode": 200 + } + }, + "description": "Guardian - Updates tenant settings", + "source_type": "gd_tenant_update" + }, + "integrations": { + "Auth0": false + }, + "originalTimestamp": "2022-10-31T06:15:25.196Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ], + "errQueue": [] + }, + "skip": "dynamic anonymousId" +} diff --git a/go/webhook/testcases/testdata/testcases/close_crm/group_creation.json b/go/webhook/testcases/testdata/testcases/close_crm/group_creation.json new file mode 100644 index 0000000000..bff58cdf04 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/close_crm/group_creation.json @@ -0,0 +1,102 @@ +{ + "name": "close_crm", + "description": "group creation", + "input": { + "request": { + "body": { + "event": { + "subscription_id": "whsub_123", + "event": { + "id": "ev_123", + "date_created": "2024-06-13T03:53:33.917000", + "date_updated": "2024-06-13T03:53:33.917000", + "organization_id": "orga_123", + "user_id": "user_123", + "request_id": "req_123", + "api_key_id": null, + "oauth_client_id": null, + "oauth_scope": null, + "object_type": "group", + "object_id": "group_123", + "lead_id": null, + "action": "created", + "changed_fields": [], + "meta": { + "request_path": "/api/v1/graphql/", + "request_method": "POST" + }, + "data": { + "id": "group_123", + "name": "Test group", + "members": [ + { + "user_id": "user_123" + } + ] + }, + "previous_data": {} + } + }, + "source": {} + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "context": { + "integration": { + "name": "CloseCRM" + }, + "library": { + "name": "unknown", + "version": "unknown" + } + }, + "event": "group created", + "integrations": { + "CloseCRM": false + }, + "messageId": "ev_123", + "originalTimestamp": "2024-06-ThT03:53:33.917+00:00", + "properties": { + "action": "created", + "data": { + "id": "group_123", + "members": [ + { + "user_id": "user_123" + } + ], + "name": "Test group" + }, + "date_created": "2024-06-13T03:53:33.917000", + "date_updated": "2024-06-13T03:53:33.917000", + "id": "ev_123", + "meta": { + "request_method": "POST", + "request_path": "/api/v1/graphql/" + }, + "object_id": "group_123", + "object_type": "group", + "organization_id": "orga_123", + "request_id": "req_123", + "subscription_id": "whsub_123", + "user_id": "user_123" + }, + "type": "track", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/close_crm/lead_deletion.json b/go/webhook/testcases/testdata/testcases/close_crm/lead_deletion.json new file mode 100644 index 0000000000..4e7076e03d --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/close_crm/lead_deletion.json @@ -0,0 +1,105 @@ +{ + "name": "close_crm", + "description": "lead deletion", + "input": { + "request": { + "body": { + "event": { + "subscription_id": "whsub_123", + "event": { + "id": "ev_123", + "date_created": "2024-06-14T05:16:04.138000", + "date_updated": "2024-06-14T05:16:04.138000", + "organization_id": "orga_123", + "user_id": "user_123", + "request_id": "req_123", + "api_key_id": "api_123", + "oauth_client_id": null, + "oauth_scope": null, + "object_type": "lead", + "object_id": "lead_123", + "lead_id": "lead_123", + "action": "deleted", + "changed_fields": [], + "meta": { + "request_path": "/api/v1/lead/lead_123/", + "request_method": "DELETE" + }, + "data": {}, + "previous_data": { + "created_by_name": "Rudder User", + "addresses": [], + "description": "", + "url": null, + "date_created": "2024-06-14T05:13:42.239000+00:00", + "status_id": "stat_123", + "contact_ids": ["cont_123"], + "id": "lead_12", + "date_updated": "2024-06-14T05:13:42.262000+00:00", + "updated_by_name": "Rudder User", + "status_label": "Potential", + "name": "test name", + "display_name": "test name", + "organization_id": "orga_123", + "updated_by": "user_123", + "created_by": "user_123" + } + } + }, + "source": {} + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "CloseCRM" + } + }, + "integrations": { + "CloseCRM": false + }, + "type": "track", + "event": "lead deleted", + "userId": "lead_123", + "messageId": "ev_123", + "originalTimestamp": "2024-06-FrT05:16:04.138+00:00", + "properties": { + "id": "ev_123", + "date_created": "2024-06-14T05:16:04.138000", + "date_updated": "2024-06-14T05:16:04.138000", + "organization_id": "orga_123", + "user_id": "user_123", + "request_id": "req_123", + "api_key_id": "api_123", + "object_type": "lead", + "object_id": "lead_123", + "lead_id": "lead_123", + "action": "deleted", + "meta": { + "request_path": "/api/v1/lead/lead_123/", + "request_method": "DELETE" + }, + "data": {}, + "subscription_id": "whsub_123" + }, + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/close_crm/lead_update.json b/go/webhook/testcases/testdata/testcases/close_crm/lead_update.json new file mode 100644 index 0000000000..186724a793 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/close_crm/lead_update.json @@ -0,0 +1,147 @@ +{ + "name": "close_crm", + "description": "lead update", + "input": { + "request": { + "body": { + "event": { + "event": { + "date_created": "2019-01-15T12:48:23.395000", + "meta": { + "request_method": "PUT", + "request_path": "/api/v1/opportunity/object_id/" + }, + "id": "ev_123", + "action": "updated", + "date_updated": "2019-01-15T12:48:23.395000", + "changed_fields": [ + "confidence", + "date_updated", + "status_id", + "status_label", + "status_type" + ], + "previous_data": { + "status_type": "active", + "confidence": 70, + "date_updated": "2019-01-15T12:47:39.873000+00:00", + "status_id": "stat_123", + "status_label": "Active" + }, + "organization_id": "orga_123", + "data": { + "contact_name": "Mr. Jones", + "user_name": "Joe Kemp", + "value_period": "one_time", + "updated_by_name": "Joe Kemp", + "date_created": "2019-01-15T12:41:24.496000+00:00", + "user_id": "user_123", + "updated_by": "user_123", + "value_currency": "USD", + "organization_id": "orga_123", + "status_label": "Won", + "contact_id": "cont_123", + "status_type": "won", + "created_by_name": "Joe Kemp", + "id": "id_12", + "lead_name": "KLine", + "date_lost": null, + "note": "", + "date_updated": "2019-01-15T12:48:23.392000+00:00", + "status_id": "stat_12", + "value": 100000, + "created_by": "user_123", + "value_formatted": "$1,000", + "date_won": "2019-01-15", + "lead_id": "lead_123", + "confidence": 100 + }, + "request_id": "req_123", + "object_id": "object_id", + "user_id": "user_123", + "object_type": "opportunity", + "lead_id": "lead_123" + }, + "subscription_id": "whsub_123" + }, + "source": {} + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "CloseCRM" + } + }, + "integrations": { + "CloseCRM": false + }, + "type": "track", + "event": "opportunity updated", + "messageId": "ev_123", + "userId": "lead_123", + "originalTimestamp": "2019-01-TuT12:48:23.395+00:00", + "properties": { + "date_created": "2019-01-15T12:48:23.395000", + "meta": { + "request_method": "PUT", + "request_path": "/api/v1/opportunity/object_id/" + }, + "id": "ev_123", + "action": "updated", + "date_updated": "2019-01-15T12:48:23.395000", + "organization_id": "orga_123", + "data": { + "contact_name": "Mr. Jones", + "user_name": "Joe Kemp", + "value_period": "one_time", + "updated_by_name": "Joe Kemp", + "date_created": "2019-01-15T12:41:24.496000+00:00", + "user_id": "user_123", + "updated_by": "user_123", + "value_currency": "USD", + "organization_id": "orga_123", + "status_label": "Won", + "contact_id": "cont_123", + "status_type": "won", + "created_by_name": "Joe Kemp", + "id": "id_12", + "lead_name": "KLine", + "note": "", + "date_updated": "2019-01-15T12:48:23.392000+00:00", + "status_id": "stat_12", + "value": 100000, + "created_by": "user_123", + "value_formatted": "$1,000", + "date_won": "2019-01-15", + "lead_id": "lead_123", + "confidence": 100 + }, + "request_id": "req_123", + "object_id": "object_id", + "user_id": "user_123", + "object_type": "opportunity", + "lead_id": "lead_123", + "subscription_id": "whsub_123" + }, + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/moengage/batch_of_events.json b/go/webhook/testcases/testdata/testcases/moengage/batch_of_events.json new file mode 100644 index 0000000000..76f72961ca --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/moengage/batch_of_events.json @@ -0,0 +1,389 @@ +{ + "name": "moengage", + "description": "Batch of events", + "input": { + "request": { + "body": { + "batch": [ + { + "type": "page", + "event": "home", + "sentAt": "2020-11-12T21:12:54.117Z", + "userId": "sajal", + "channel": "mobile", + "context": { + "traits": {}, + "library": { + "name": "rudder-sdk-ruby-sync", + "version": "1.0.7" + }, + "page": { + "path": "/Rectified.html", + "referrer": "http://localhost:1112/", + "search": "", + "title": "", + "url": "http://localhost:1112/Rectified.html" + }, + "userAgent": "Dalvik/2.1.0 (Linux; U; Android 10; Redmi K20 Pro MIUI/V12.0.3.0.QFKINXM)" + }, + "rudderId": "asdfasdfsadf", + "properties": { + "name": "asdfsadf" + }, + "timestamp": "2020-11-12T21:12:41.320Z", + "anonymousId": "123123123123" + }, + { + "anonymousId": "4eb021e9-a2af-4926-ae82-fe996d12f3c5", + "channel": "web", + "context": { + "timezone": "Asia/Tokyo", + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.1.6" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.1.6" + }, + "locale": "en-GB", + "os": { + "name": "", + "version": "" + }, + "page": { + "path": "/testing/script-test.html", + "referrer": "", + "search": "", + "title": "", + "url": "http://localhost:3243/testing/script-test.html" + }, + "screen": { + "density": 2 + }, + "traits": { + "company": { + "id": "abc123" + }, + "createdAt": "Thu Mar 24 2016 17:46:45 GMT+0000 (UTC)", + "email": "rudderTest@gmail.com", + "name": "Rudder Test", + "plan": "Enterprise" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.80 Safari/537.36" + }, + "event": "Order Completed", + "integrations": { + "All": true + }, + "messageId": "a0adfab9-baf7-4e09-a2ce-bbe2844c324a", + "originalTimestamp": "2020-10-16T08:10:12.782Z", + "properties": { + "checkout_id": "what is checkout id here??", + "coupon": "APPARELSALE", + "currency": "GBP", + "order_id": "transactionId", + "category": "some category", + "originalArray": [ + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + } + ], + "products": [ + { + "brand": "", + "category": "Merch", + "currency": "GBP", + "image_url": "https://www.example.com/product/bacon-jam.jpg", + "name": "Food/Drink", + "position": 1, + "price": 3, + "product_id": "product-bacon-jam", + "quantity": 2, + "sku": "sku-1", + "typeOfProduct": "Food", + "url": "https://www.example.com/product/bacon-jam", + "value": 6, + "variant": "Extra topped" + }, + { + "brand": "Levis", + "category": "Merch", + "currency": "GBP", + "image_url": "https://www.example.com/product/t-shirt.jpg", + "name": "T-Shirt", + "position": 2, + "price": 12.99, + "product_id": "product-t-shirt", + "quantity": 1, + "sku": "sku-2", + "typeOfProduct": "Shirt", + "url": "https://www.example.com/product/t-shirt", + "value": 12.99, + "variant": "White" + }, + { + "brand": "Levis", + "category": "Merch", + "coupon": "APPARELSALE", + "currency": "GBP", + "image_url": "https://www.example.com/product/offer-t-shirt.jpg", + "name": "T-Shirt-on-offer", + "position": 1, + "price": 12.99, + "product_id": "offer-t-shirt", + "quantity": 1, + "sku": "sku-3", + "typeOfProduct": "Shirt", + "url": "https://www.example.com/product/offer-t-shirt", + "value": 12.99, + "variant": "Black" + } + ], + "revenue": 31.98, + "shipping": 4, + "value": 31.98 + }, + "receivedAt": "2020-10-16T13:40:12.792+05:30", + "request_ip": "[::1]", + "sentAt": "2020-10-16T08:10:12.783Z", + "timestamp": "2020-10-16T13:40:12.791+05:30", + "type": "track", + "userId": "rudder123" + } + ] + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "type": "page", + "event": "home", + "sentAt": "2020-11-12T21:12:54.117Z", + "userId": "sajal", + "channel": "mobile", + "context": { + "traits": {}, + "library": { + "name": "rudder-sdk-ruby-sync", + "version": "1.0.7" + }, + "page": { + "path": "/Rectified.html", + "referrer": "http://localhost:1112/", + "search": "", + "title": "", + "url": "http://localhost:1112/Rectified.html" + }, + "userAgent": "Dalvik/2.1.0 (Linux; U; Android 10; Redmi K20 Pro MIUI/V12.0.3.0.QFKINXM)" + }, + "rudderId": "asdfasdfsadf", + "properties": { + "name": "asdfsadf" + }, + "timestamp": "2020-11-12T21:12:41.320Z", + "anonymousId": "123123123123", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + }, + { + "anonymousId": "4eb021e9-a2af-4926-ae82-fe996d12f3c5", + "channel": "web", + "context": { + "timezone": "Asia/Tokyo", + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.1.6" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.1.6" + }, + "locale": "en-GB", + "os": { + "name": "", + "version": "" + }, + "page": { + "path": "/testing/script-test.html", + "referrer": "", + "search": "", + "title": "", + "url": "http://localhost:3243/testing/script-test.html" + }, + "screen": { + "density": 2 + }, + "traits": { + "company": { + "id": "abc123" + }, + "createdAt": "Thu Mar 24 2016 17:46:45 GMT+0000 (UTC)", + "email": "rudderTest@gmail.com", + "name": "Rudder Test", + "plan": "Enterprise" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.80 Safari/537.36" + }, + "event": "Order Completed", + "integrations": { + "All": true + }, + "messageId": "a0adfab9-baf7-4e09-a2ce-bbe2844c324a", + "originalTimestamp": "2020-10-16T08:10:12.782Z", + "properties": { + "checkout_id": "what is checkout id here??", + "coupon": "APPARELSALE", + "currency": "GBP", + "order_id": "transactionId", + "category": "some category", + "originalArray": [ + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + }, + { + "nested_field": "nested value", + "tags": ["tag_1", "tag_2", "tag_3"] + } + ], + "products": [ + { + "brand": "", + "category": "Merch", + "currency": "GBP", + "image_url": "https://www.example.com/product/bacon-jam.jpg", + "name": "Food/Drink", + "position": 1, + "price": 3, + "product_id": "product-bacon-jam", + "quantity": 2, + "sku": "sku-1", + "typeOfProduct": "Food", + "url": "https://www.example.com/product/bacon-jam", + "value": 6, + "variant": "Extra topped" + }, + { + "brand": "Levis", + "category": "Merch", + "currency": "GBP", + "image_url": "https://www.example.com/product/t-shirt.jpg", + "name": "T-Shirt", + "position": 2, + "price": 12.99, + "product_id": "product-t-shirt", + "quantity": 1, + "sku": "sku-2", + "typeOfProduct": "Shirt", + "url": "https://www.example.com/product/t-shirt", + "value": 12.99, + "variant": "White" + }, + { + "brand": "Levis", + "category": "Merch", + "coupon": "APPARELSALE", + "currency": "GBP", + "image_url": "https://www.example.com/product/offer-t-shirt.jpg", + "name": "T-Shirt-on-offer", + "position": 1, + "price": 12.99, + "product_id": "offer-t-shirt", + "quantity": 1, + "sku": "sku-3", + "typeOfProduct": "Shirt", + "url": "https://www.example.com/product/offer-t-shirt", + "value": 12.99, + "variant": "Black" + } + ], + "revenue": 31.98, + "shipping": 4, + "value": 31.98 + }, + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "sentAt": "2020-10-16T08:10:12.783Z", + "timestamp": "2020-10-16T13:40:12.791+05:30", + "type": "track", + "userId": "rudder123" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/moengage/simple_track_call.json b/go/webhook/testcases/testdata/testcases/moengage/simple_track_call.json new file mode 100644 index 0000000000..f9b11bffb4 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/moengage/simple_track_call.json @@ -0,0 +1,246 @@ +{ + "name": "moengage", + "description": "Simple track call", + "input": { + "request": { + "body": { + "anonymousId": "4eb021e9-a2af-4926-ae82-fe996d12f3c5", + "channel": "web", + "context": { + "timezone": "Wrong/Timezone", + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.1.6" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.1.6" + }, + "locale": "en-GB", + "os": { + "name": "", + "version": "" + }, + "page": { + "path": "/testing/script-test.html", + "referrer": "", + "search": "", + "title": "", + "url": "http://localhost:3243/testing/script-test.html" + }, + "screen": { + "density": 2 + }, + "traits": { + "company": { + "id": "abc123" + }, + "createdAt": "Thu Mar 24 2016 17:46:45 GMT+0000 (UTC)", + "email": "rudderTest@gmail.com", + "name": "Rudder Test", + "plan": "Enterprise" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.80 Safari/537.36" + }, + "event": "Order Completed", + "integrations": { + "All": true + }, + "messageId": "a0adfab9-baf7-4e09-a2ce-bbe2844c324a", + "originalTimestamp": "2020-10-16T08:10:12.782Z", + "properties": { + "checkout_id": "what is checkout id here??", + "coupon": "APPARELSALE", + "currency": "GBP", + "order_id": "transactionId", + "products": [ + { + "brand": "", + "category": "Merch", + "currency": "GBP", + "image_url": "https://www.example.com/product/bacon-jam.jpg", + "name": "Food/Drink", + "position": 1, + "price": 3, + "product_id": "product-bacon-jam", + "quantity": 2, + "sku": "sku-1", + "typeOfProduct": "Food", + "url": "https://www.example.com/product/bacon-jam", + "value": 6, + "variant": "Extra topped" + }, + { + "brand": "Levis", + "category": "Merch", + "currency": "GBP", + "image_url": "https://www.example.com/product/t-shirt.jpg", + "name": "T-Shirt", + "position": 2, + "price": 12.99, + "product_id": "product-t-shirt", + "quantity": 1, + "sku": "sku-2", + "typeOfProduct": "Shirt", + "url": "https://www.example.com/product/t-shirt", + "value": 12.99, + "variant": "White" + }, + { + "brand": "Levis", + "category": "Merch", + "coupon": "APPARELSALE", + "currency": "GBP", + "image_url": "https://www.example.com/product/offer-t-shirt.jpg", + "name": "T-Shirt-on-offer", + "position": 1, + "price": 12.99, + "product_id": "offer-t-shirt", + "quantity": 1, + "sku": "sku-3", + "typeOfProduct": "Shirt", + "url": "https://www.example.com/product/offer-t-shirt", + "value": 12.99, + "variant": "Black" + } + ], + "revenue": 31.98, + "shipping": 4, + "value": 31.98 + }, + "sentAt": "2020-10-16T08:10:12.783Z", + "timestamp": "2020-10-16T13:40:12.791+05:30", + "type": "track", + "userId": "rudder123" + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "anonymousId": "4eb021e9-a2af-4926-ae82-fe996d12f3c5", + "channel": "web", + "context": { + "timezone": "Wrong/Timezone", + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.1.6" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.1.6" + }, + "locale": "en-GB", + "os": { + "name": "", + "version": "" + }, + "page": { + "path": "/testing/script-test.html", + "referrer": "", + "search": "", + "title": "", + "url": "http://localhost:3243/testing/script-test.html" + }, + "screen": { + "density": 2 + }, + "traits": { + "company": { + "id": "abc123" + }, + "createdAt": "Thu Mar 24 2016 17:46:45 GMT+0000 (UTC)", + "email": "rudderTest@gmail.com", + "name": "Rudder Test", + "plan": "Enterprise" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.80 Safari/537.36" + }, + "event": "Order Completed", + "integrations": { + "All": true + }, + "messageId": "a0adfab9-baf7-4e09-a2ce-bbe2844c324a", + "originalTimestamp": "2020-10-16T08:10:12.782Z", + "properties": { + "checkout_id": "what is checkout id here??", + "coupon": "APPARELSALE", + "currency": "GBP", + "order_id": "transactionId", + "products": [ + { + "brand": "", + "category": "Merch", + "currency": "GBP", + "image_url": "https://www.example.com/product/bacon-jam.jpg", + "name": "Food/Drink", + "position": 1, + "price": 3, + "product_id": "product-bacon-jam", + "quantity": 2, + "sku": "sku-1", + "typeOfProduct": "Food", + "url": "https://www.example.com/product/bacon-jam", + "value": 6, + "variant": "Extra topped" + }, + { + "brand": "Levis", + "category": "Merch", + "currency": "GBP", + "image_url": "https://www.example.com/product/t-shirt.jpg", + "name": "T-Shirt", + "position": 2, + "price": 12.99, + "product_id": "product-t-shirt", + "quantity": 1, + "sku": "sku-2", + "typeOfProduct": "Shirt", + "url": "https://www.example.com/product/t-shirt", + "value": 12.99, + "variant": "White" + }, + { + "brand": "Levis", + "category": "Merch", + "coupon": "APPARELSALE", + "currency": "GBP", + "image_url": "https://www.example.com/product/offer-t-shirt.jpg", + "name": "T-Shirt-on-offer", + "position": 1, + "price": 12.99, + "product_id": "offer-t-shirt", + "quantity": 1, + "sku": "sku-3", + "typeOfProduct": "Shirt", + "url": "https://www.example.com/product/offer-t-shirt", + "value": 12.99, + "variant": "Black" + } + ], + "revenue": 31.98, + "shipping": 4, + "value": 31.98 + }, + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "sentAt": "2020-10-16T08:10:12.783Z", + "timestamp": "2020-10-16T13:40:12.791+05:30", + "type": "track", + "userId": "rudder123" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/ortto/simple_track_call.json b/go/webhook/testcases/testdata/testcases/ortto/simple_track_call.json new file mode 100644 index 0000000000..81e7241355 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/ortto/simple_track_call.json @@ -0,0 +1,108 @@ +{ + "name": "ortto", + "description": "Simple track call", + "input": { + "request": { + "body": { + "activity": { + "id": "00651b946bfef7e80478efee", + "field_id": "act::s-all", + "created": "2023-10-03T04:11:23Z", + "attr": { + "str::is": "API", + "str::s-ctx": "Subscribed via API" + } + }, + "contact": { + "external_id": "user_x", + "city": { + "name": "Kolkata", + "id": 0, + "lat": 37751000, + "lng": -97822000 + }, + "country": { + "name": "United States", + "id": 6252001, + "lat": 0, + "lng": 0 + }, + "email": "xyz@email.com", + "first_name": "Ujjwal", + "last_name": "Ujjwal", + "birthday": { + "year": 1980, + "month": 12, + "day": 11, + "timezone": "Australia/Sydney" + }, + "phone_number": { + "c": "91", + "n": "401234567" + } + }, + "id": "00651b946cef87c7af64f4f3", + "time": "2023-10-03T04:11:24.25726779Z", + "webhook_id": "651b8aec8002153e16319fd3" + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "userId": "user_x", + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "ortto" + }, + "traits": { + "email": "xyz@email.com", + "birthday": "1980-12-11", + "firstName": "Ujjwal", + "lastName": "Ujjwal", + "phone": "91401234567", + "address": { + "city": "Kolkata", + "country": "United States" + } + } + }, + "event": "Resubscribe globally", + "integrations": { + "ortto": false + }, + "type": "track", + "messageId": "00651b946cef87c7af64f4f3", + "originalTimestamp": "2023-10-03T04:11:24.000Z", + "properties": { + "activity.id": "00651b946bfef7e80478efee", + "activity.created": "2023-10-03T04:11:23Z", + "activity.attr.str::is": "API", + "activity.attr.str::s-ctx": "Subscribed via API", + "contact.birthday.timezone": "Australia/Sydney", + "contact.city.id": 0, + "contact.city.lat": 37751000, + "contact.city.lng": -97822000, + "contact.country.id": 6252001, + "contact.country.lat": 0, + "contact.country.lng": 0, + "webhook_id": "651b8aec8002153e16319fd3" + }, + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/ortto/simple_track_call_with_unknown_field_id.json b/go/webhook/testcases/testdata/testcases/ortto/simple_track_call_with_unknown_field_id.json new file mode 100644 index 0000000000..6ee5ab4237 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/ortto/simple_track_call_with_unknown_field_id.json @@ -0,0 +1,104 @@ +{ + "name": "ortto", + "description": "Simple track call with unknown field id", + "input": { + "request": { + "body": { + "activity": { + "id": "00651b946bfef7e80478efee", + "field_id": "act::test_webhook", + "created": "2023-10-03T04:11:23Z", + "attr": { + "str::is": "API", + "str::s-ctx": "Subscribed via API" + } + }, + "contact": { + "external_id": "user_x", + "city": { + "name": "Kolkata", + "id": 0, + "lat": 37751000, + "lng": -97822000 + }, + "contact_id": "006524f0b8d370050056e400", + "country": { + "name": "United States", + "id": 6252001, + "lat": 0, + "lng": 0 + }, + "email": "xyz@email.com", + "first_name": "Ujjwal", + "last_name": "Ujjwal", + "birthday": { + "year": 1980, + "month": 3, + "day": 4, + "timezone": "Australia/Sydney" + }, + "phone_number": { + "c": "91", + "n": "401234567" + } + }, + "id": "00651b946cef87c7af64f4f3", + "time": "2023-10-03T04:11:24.25726779Z", + "webhook_id": "651b8aec8002153e16319fd3" + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": { + "activity": { + "id": "00651b946bfef7e80478efee", + "field_id": "act::test_webhook", + "created": "2023-10-03T04:11:23Z", + "attr": { + "str::is": "API", + "str::s-ctx": "Subscribed via API" + } + }, + "contact": { + "external_id": "user_x", + "city": { + "name": "Kolkata", + "id": 0, + "lat": 37751000, + "lng": -97822000 + }, + "contact_id": "006524f0b8d370050056e400", + "country": { + "name": "United States", + "id": 6252001, + "lat": 0, + "lng": 0 + }, + "email": "xyz@email.com", + "first_name": "Ujjwal", + "last_name": "Ujjwal", + "birthday": { + "year": 1980, + "month": 3, + "day": 4, + "timezone": "Australia/Sydney" + }, + "phone_number": { + "c": "91", + "n": "401234567" + } + }, + "id": "00651b946cef87c7af64f4f3", + "time": "2023-10-03T04:11:24.25726779Z", + "webhook_id": "651b8aec8002153e16319fd3" + } + }, + "queue": [], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/revenuecat/initial_purchase_event.json b/go/webhook/testcases/testdata/testcases/revenuecat/initial_purchase_event.json new file mode 100644 index 0000000000..6dbcbd6622 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/revenuecat/initial_purchase_event.json @@ -0,0 +1,121 @@ +{ + "name": "revenuecat", + "description": "Initial purchase event", + "input": { + "request": { + "body": { + "api_version": "1.0", + "event": { + "aliases": ["yourCustomerAliasedID", "yourCustomerAliasedID"], + "app_id": "yourAppID", + "app_user_id": "yourCustomerAppUserID", + "commission_percentage": 0.3, + "country_code": "US", + "currency": "USD", + "entitlement_id": "pro_cat", + "entitlement_ids": ["pro_cat"], + "environment": "PRODUCTION", + "event_timestamp_ms": 1591121855319, + "expiration_at_ms": 1591726653000, + "id": "UniqueIdentifierOfEvent", + "is_family_share": false, + "offer_code": "free_month", + "original_app_user_id": "OriginalAppUserID", + "original_transaction_id": "1530648507000", + "period_type": "NORMAL", + "presented_offering_id": "OfferingID", + "price": 2.49, + "price_in_purchased_currency": 2.49, + "product_id": "onemonth_no_trial", + "purchased_at_ms": 1591121853000, + "store": "APP_STORE", + "subscriber_attributes": { + "$Favorite Cat": { + "updated_at_ms": 1581121853000, + "value": "Garfield" + } + }, + "takehome_percentage": 0.7, + "tax_percentage": 0.3, + "transaction_id": "170000869511114", + "type": "INITIAL_PURCHASE" + } + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "RevenueCat" + }, + "externalId": [ + { + "type": "revenuecatAppUserId", + "id": "yourCustomerAppUserID" + } + ] + }, + "integrations": { + "RevenueCat": false + }, + "type": "track", + "properties": { + "aliases": ["yourCustomerAliasedID", "yourCustomerAliasedID"], + "appId": "yourAppID", + "appUserId": "yourCustomerAppUserID", + "commissionPercentage": 0.3, + "countryCode": "US", + "currency": "USD", + "entitlementId": "pro_cat", + "entitlementIds": ["pro_cat"], + "environment": "PRODUCTION", + "eventTimestampMs": 1591121855319, + "expirationAtMs": 1591726653000, + "id": "UniqueIdentifierOfEvent", + "isFamilyShare": false, + "offerCode": "free_month", + "originalAppUserId": "OriginalAppUserID", + "originalTransactionId": "1530648507000", + "periodType": "NORMAL", + "presentedOfferingId": "OfferingID", + "price": 2.49, + "priceInPurchasedCurrency": 2.49, + "productId": "onemonth_no_trial", + "purchasedAtMs": 1591121853000, + "store": "APP_STORE", + "subscriberAttributes": { + "$Favorite Cat": { + "updated_at_ms": 1581121853000, + "value": "Garfield" + } + }, + "takehomePercentage": 0.7, + "taxPercentage": 0.3, + "transactionId": "170000869511114", + "type": "INITIAL_PURCHASE" + }, + "event": "INITIAL_PURCHASE", + "userId": "yourCustomerAppUserID", + "messageId": "UniqueIdentifierOfEvent", + "originalTimestamp": "2020-06-02T18:17:35.319Z", + "sentAt": "2020-06-02T18:17:35.319Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/revenuecat/purchase_event_with_anonymous_user.json b/go/webhook/testcases/testdata/testcases/revenuecat/purchase_event_with_anonymous_user.json new file mode 100644 index 0000000000..12e2f4ba11 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/revenuecat/purchase_event_with_anonymous_user.json @@ -0,0 +1,112 @@ +{ + "name": "revenuecat", + "description": "Purchase event with anonymous user", + "input": { + "request": { + "body": { + "api_version": "1.0", + "event": { + "aliases": ["yourCustomerAliasedID", "yourCustomerAliasedID"], + "app_id": "yourAppID", + "commission_percentage": 0.3, + "country_code": "US", + "currency": "USD", + "entitlement_id": "pro_cat", + "entitlement_ids": ["pro_cat"], + "environment": "PRODUCTION", + "event_timestamp_ms": 1591121855319, + "expiration_at_ms": 1591726653000, + "id": "UniqueIdentifierOfEvent", + "is_family_share": false, + "offer_code": "free_month", + "original_transaction_id": "1530648507000", + "period_type": "NORMAL", + "presented_offering_id": "OfferingID", + "price": 2.49, + "price_in_purchased_currency": 2.49, + "product_id": "onemonth_no_trial", + "purchased_at_ms": 1591121853000, + "store": "APP_STORE", + "subscriber_attributes": { + "$Favorite Cat": { + "updated_at_ms": 1581121853000, + "value": "Garfield" + } + }, + "takehome_percentage": 0.7, + "tax_percentage": 0.3, + "transaction_id": "170000869511114", + "type": "INITIAL_PURCHASE" + } + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "RevenueCat" + } + }, + "integrations": { + "RevenueCat": false + }, + "type": "track", + "properties": { + "aliases": ["yourCustomerAliasedID", "yourCustomerAliasedID"], + "appId": "yourAppID", + "commissionPercentage": 0.3, + "countryCode": "US", + "currency": "USD", + "entitlementId": "pro_cat", + "entitlementIds": ["pro_cat"], + "environment": "PRODUCTION", + "eventTimestampMs": 1591121855319, + "expirationAtMs": 1591726653000, + "id": "UniqueIdentifierOfEvent", + "isFamilyShare": false, + "offerCode": "free_month", + "originalTransactionId": "1530648507000", + "periodType": "NORMAL", + "presentedOfferingId": "OfferingID", + "price": 2.49, + "priceInPurchasedCurrency": 2.49, + "productId": "onemonth_no_trial", + "purchasedAtMs": 1591121853000, + "store": "APP_STORE", + "subscriberAttributes": { + "$Favorite Cat": { + "updated_at_ms": 1581121853000, + "value": "Garfield" + } + }, + "takehomePercentage": 0.7, + "taxPercentage": 0.3, + "transactionId": "170000869511114", + "type": "INITIAL_PURCHASE" + }, + "event": "INITIAL_PURCHASE", + "userId": "", + "anonymousId": "97fcd7b2-cc24-47d7-b776-057b7b199513", + "messageId": "UniqueIdentifierOfEvent", + "originalTimestamp": "2020-06-02T18:17:35.319Z", + "sentAt": "2020-06-02T18:17:35.319Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/revenuecat/simple_track_call.json b/go/webhook/testcases/testdata/testcases/revenuecat/simple_track_call.json new file mode 100644 index 0000000000..851537141b --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/revenuecat/simple_track_call.json @@ -0,0 +1,149 @@ +{ + "name": "revenuecat", + "description": "Simple track call", + "input": { + "request": { + "body": { + "api_version": "1.0", + "event": { + "aliases": [ + "f8e14f51-0c76-49ba-8d67-c229f1875dd9", + "389ad6dd-bb40-4c03-9471-1353da2d55ec" + ], + "app_user_id": "f8e14f51-0c76-49ba-8d67-c229f1875dd9", + "commission_percentage": null, + "country_code": "US", + "currency": null, + "entitlement_id": null, + "entitlement_ids": null, + "environment": "SANDBOX", + "event_timestamp_ms": 1698617217232, + "expiration_at_ms": 1698624417232, + "id": "8CF0CD6C-CAF3-41FB-968A-661938235AF0", + "is_family_share": null, + "offer_code": null, + "original_app_user_id": "f8e14f51-0c76-49ba-8d67-c229f1875dd9", + "original_transaction_id": null, + "period_type": "NORMAL", + "presented_offering_id": null, + "price": null, + "price_in_purchased_currency": null, + "product_id": "test_product", + "purchased_at_ms": 1698617217232, + "store": "APP_STORE", + "subscriber_attributes": { + "$displayName": { + "updated_at_ms": 1698617217232, + "value": "Mister Mistoffelees" + }, + "$email": { + "updated_at_ms": 1698617217232, + "value": "tuxedo@revenuecat.com" + }, + "$phoneNumber": { + "updated_at_ms": 1698617217232, + "value": "+19795551234" + }, + "my_custom_attribute_1": { + "updated_at_ms": 1698617217232, + "value": "catnip" + } + }, + "takehome_percentage": null, + "tax_percentage": null, + "transaction_id": null, + "type": "TEST" + } + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "RevenueCat" + }, + "externalId": [ + { + "type": "revenuecatAppUserId", + "id": "f8e14f51-0c76-49ba-8d67-c229f1875dd9" + } + ] + }, + "integrations": { + "RevenueCat": false + }, + "type": "track", + "properties": { + "aliases": [ + "f8e14f51-0c76-49ba-8d67-c229f1875dd9", + "389ad6dd-bb40-4c03-9471-1353da2d55ec" + ], + "appUserId": "f8e14f51-0c76-49ba-8d67-c229f1875dd9", + "commissionPercentage": null, + "countryCode": "US", + "currency": null, + "entitlementId": null, + "entitlementIds": null, + "environment": "SANDBOX", + "eventTimestampMs": 1698617217232, + "expirationAtMs": 1698624417232, + "id": "8CF0CD6C-CAF3-41FB-968A-661938235AF0", + "isFamilyShare": null, + "offerCode": null, + "originalAppUserId": "f8e14f51-0c76-49ba-8d67-c229f1875dd9", + "originalTransactionId": null, + "periodType": "NORMAL", + "presentedOfferingId": null, + "price": null, + "priceInPurchasedCurrency": null, + "productId": "test_product", + "purchasedAtMs": 1698617217232, + "store": "APP_STORE", + "subscriberAttributes": { + "$displayName": { + "updated_at_ms": 1698617217232, + "value": "Mister Mistoffelees" + }, + "$email": { + "updated_at_ms": 1698617217232, + "value": "tuxedo@revenuecat.com" + }, + "$phoneNumber": { + "updated_at_ms": 1698617217232, + "value": "+19795551234" + }, + "my_custom_attribute_1": { + "updated_at_ms": 1698617217232, + "value": "catnip" + } + }, + "takehomePercentage": null, + "taxPercentage": null, + "transactionId": null, + "type": "TEST" + }, + "event": "TEST", + "userId": "f8e14f51-0c76-49ba-8d67-c229f1875dd9", + "messageId": "8CF0CD6C-CAF3-41FB-968A-661938235AF0", + "originalTimestamp": "2023-10-29T22:06:57.232Z", + "sentAt": "2023-10-29T22:06:57.232Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/shopify/carts_create.json b/go/webhook/testcases/testdata/testcases/shopify/carts_create.json new file mode 100644 index 0000000000..6c37baa443 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/shopify/carts_create.json @@ -0,0 +1,16 @@ +{ + "name": "shopify", + "description": "Track Call -> carts_create ", + "input": { + "request": { + "query": "topic=carts_create" + } + }, + "output": { + "response": { + "status": 200, + "contentType": "text/plain", + "body": "OK" + } + } +} diff --git a/go/webhook/testcases/testdata/testcases/shopify/fullfillments_updated_event.json b/go/webhook/testcases/testdata/testcases/shopify/fullfillments_updated_event.json new file mode 100644 index 0000000000..a7d9717c3d --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/shopify/fullfillments_updated_event.json @@ -0,0 +1,251 @@ +{ + "name": "shopify", + "description": "Fulfillment updated event", + "input": { + "request": { + "query": "signature=rudderstack&topic=fulfillments_update", + "body": { + "shipping_address": { + "address1": "11 Rani Sankari Lane Patuapara Bhowanipore" + }, + "billing_address": { + "address1": "11 Rani Sankari Lane Patuapara Bhowanipore" + }, + "admin_graphql_api_id": "gid://shopify/Fulfillment/4124667937024", + "created_at": "2022-01-05T18:13:02+05:30", + "destination": null, + "email": "test_person@email.com", + "id": 4124667937024, + "line_items": [ + { + "admin_graphql_api_id": "gid://shopify/LineItem/11896203149568", + "discount_allocations": [], + "duties": [], + "fulfillable_quantity": 0, + "fulfillment_service": "manual", + "fulfillment_status": "fulfilled", + "gift_card": false, + "grams": 0, + "id": 11896203149568, + "name": "p1", + "origin_location": { + "address1": "74 CC/7, Anupama Housing Estate - II", + "address2": "", + "city": "Kolkatta", + "country_code": "IN", + "id": 3373642219776, + "name": "74 CC/7, Anupama Housing Estate - II", + "province_code": "WB", + "zip": "700052" + }, + "price": "5000.00", + "price_set": { + "presentment_money": { + "amount": "5000.00", + "currency_code": "INR" + }, + "shop_money": { + "amount": "5000.00", + "currency_code": "INR" + } + }, + "product_exists": true, + "product_id": 7510929801472, + "properties": [], + "quantity": 1, + "requires_shipping": true, + "sku": "15", + "tax_lines": [ + { + "channel_liable": false, + "price": "900.00", + "price_set": { + "presentment_money": { + "amount": "900.00", + "currency_code": "INR" + }, + "shop_money": { + "amount": "900.00", + "currency_code": "INR" + } + }, + "rate": 0.18, + "title": "IGST" + } + ], + "taxable": true, + "title": "p1", + "total_discount": "0.00", + "total_discount_set": { + "presentment_money": { + "amount": "0.00", + "currency_code": "INR" + }, + "shop_money": { + "amount": "0.00", + "currency_code": "INR" + } + }, + "variant_id": 42211160228096, + "variant_inventory_management": "shopify", + "variant_title": "", + "vendor": "rudderstack-store" + } + ], + "location_id": 66855371008, + "name": "#1002.1", + "order_id": 4617255092480, + "origin_address": null, + "receipt": {}, + "service": "manual", + "shipment_status": null, + "status": "success", + "tracking_company": "Amazon Logistics UK", + "tracking_number": "Sample001test", + "tracking_numbers": ["Sample001test"], + "tracking_url": "https://www.amazon.co.uk/gp/help/customer/display.html?nodeId=201910530", + "tracking_urls": [ + "https://www.amazon.co.uk/gp/help/customer/display.html?nodeId=201910530" + ], + "updated_at": "2022-01-05T18:16:48+05:30" + } + } + }, + "output": { + "response": { + "status": 200, + "contentType": "text/plain", + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "RudderStack Shopify Cloud", + "version": "1.0.0" + }, + "integration": { + "name": "SHOPIFY" + }, + "topic": "fulfillments_update" + }, + "integrations": { + "SHOPIFY": true + }, + "type": "track", + "userId": "shopify-admin", + "event": "Fulfillments Update", + "properties": { + "admin_graphql_api_id": "gid://shopify/Fulfillment/4124667937024", + "created_at": "2022-01-05T18:13:02+05:30", + "destination": null, + "email": "test_person@email.com", + "id": 4124667937024, + "location_id": 66855371008, + "name": "#1002.1", + "order_id": 4617255092480, + "origin_address": null, + "receipt": {}, + "service": "manual", + "shipment_status": null, + "status": "success", + "tracking_company": "Amazon Logistics UK", + "tracking_number": "Sample001test", + "tracking_numbers": ["Sample001test"], + "tracking_url": "https://www.amazon.co.uk/gp/help/customer/display.html?nodeId=201910530", + "tracking_urls": [ + "https://www.amazon.co.uk/gp/help/customer/display.html?nodeId=201910530" + ], + "updated_at": "2022-01-05T18:16:48+05:30", + "products": [ + { + "product_id": 7510929801472, + "sku": "15", + "title": "p1", + "price": "5000.00", + "brand": "rudderstack-store", + "quantity": 1, + "admin_graphql_api_id": "gid://shopify/LineItem/11896203149568", + "discount_allocations": [], + "duties": [], + "fulfillable_quantity": 0, + "fulfillment_service": "manual", + "fulfillment_status": "fulfilled", + "gift_card": false, + "grams": 0, + "id": 11896203149568, + "origin_location": { + "address1": "74 CC/7, Anupama Housing Estate - II", + "address2": "", + "city": "Kolkatta", + "country_code": "IN", + "id": 3373642219776, + "name": "74 CC/7, Anupama Housing Estate - II", + "province_code": "WB", + "zip": "700052" + }, + "price_set": { + "presentment_money": { + "amount": "5000.00", + "currency_code": "INR" + }, + "shop_money": { + "amount": "5000.00", + "currency_code": "INR" + } + }, + "product_exists": true, + "properties": [], + "requires_shipping": true, + "tax_lines": [ + { + "channel_liable": false, + "price": "900.00", + "price_set": { + "presentment_money": { + "amount": "900.00", + "currency_code": "INR" + }, + "shop_money": { + "amount": "900.00", + "currency_code": "INR" + } + }, + "rate": 0.18, + "title": "IGST" + } + ], + "taxable": true, + "total_discount": "0.00", + "total_discount_set": { + "presentment_money": { + "amount": "0.00", + "currency_code": "INR" + }, + "shop_money": { + "amount": "0.00", + "currency_code": "INR" + } + }, + "variant_inventory_management": "shopify", + "variant": "42211160228096 " + } + ] + }, + "traits": { + "shippingAddress": { + "address1": "11 Rani Sankari Lane Patuapara Bhowanipore" + }, + "billingAddress": { + "address1": "11 Rani Sankari Lane Patuapara Bhowanipore" + }, + "email": "test_person@email.com" + }, + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "rudderId": "f4ff065d-11d0-4411-b6be-1d0540687f04", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ] + } +} diff --git a/go/webhook/testcases/testdata/testcases/shopify/identify_call.json b/go/webhook/testcases/testdata/testcases/shopify/identify_call.json new file mode 100644 index 0000000000..90c741862a --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/shopify/identify_call.json @@ -0,0 +1,174 @@ +{ + "name": "shopify", + "description": "Identify Call for customers create event", + "input": { + "request": { + "query": "topic=customers_create&signature=rudderstack", + "body": { + "id": 5747017285820, + "email": "anuraj@rudderstack.com", + "accepts_marketing": false, + "created_at": "2021-12-29T15:15:19+05:30", + "updated_at": "2021-12-29T15:15:20+05:30", + "first_name": "Anuraj", + "last_name": "Guha", + "orders_count": 0, + "state": "disabled", + "total_spent": "0.00", + "last_order_id": null, + "note": "", + "verified_email": true, + "multipass_identifier": null, + "tax_exempt": false, + "phone": "+919876543210", + "tags": "", + "last_order_name": null, + "currency": "INR", + "addresses": [ + { + "id": 6947581821116, + "customer_id": 5747017285820, + "first_name": "Anuraj", + "last_name": "Guha", + "company": "Rudderstack", + "address1": "Home", + "address2": "Apartment", + "city": "Kolkata", + "province": "West Bengal", + "country": "India", + "zip": "708091", + "phone": "+919876543210", + "name": "Anuraj Guha", + "province_code": "WB", + "country_code": "IN", + "country_name": "India", + "default": true + } + ], + "accepts_marketing_updated_at": "2021-12-29T15:15:20+05:30", + "marketing_opt_in_level": null, + "tax_exemptions": [], + "sms_marketing_consent": { + "state": "not_subscribed", + "opt_in_level": "single_opt_in", + "consent_updated_at": null, + "consent_collected_from": "SHOPIFY" + }, + "admin_graphql_api_id": "gid://shopify/Customer/5747017285820", + "default_address": { + "id": 6947581821116, + "customer_id": 5747017285820, + "first_name": "Anuraj", + "last_name": "Guha", + "company": "Rudderstack", + "address1": "Home", + "address2": "Apartment", + "city": "Kolkata", + "province": "West Bengal", + "country": "India", + "zip": "708091", + "phone": "+919876543210", + "name": "Anuraj Guha", + "province_code": "WB", + "country_code": "IN", + "country_name": "India", + "default": true + } + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "RudderStack Shopify Cloud", + "version": "1.0.0" + }, + "integration": { + "name": "SHOPIFY" + }, + "topic": "customers_create" + }, + "integrations": { + "SHOPIFY": true + }, + "type": "identify", + "userId": "5747017285820", + "traits": { + "email": "anuraj@rudderstack.com", + "firstName": "Anuraj", + "lastName": "Guha", + "phone": "+919876543210", + "addressList": [ + { + "id": 6947581821116, + "customer_id": 5747017285820, + "first_name": "Anuraj", + "last_name": "Guha", + "company": "Rudderstack", + "address1": "Home", + "address2": "Apartment", + "city": "Kolkata", + "province": "West Bengal", + "country": "India", + "zip": "708091", + "phone": "+919876543210", + "name": "Anuraj Guha", + "province_code": "WB", + "country_code": "IN", + "country_name": "India", + "default": true + } + ], + "address": { + "id": 6947581821116, + "customer_id": 5747017285820, + "first_name": "Anuraj", + "last_name": "Guha", + "company": "Rudderstack", + "address1": "Home", + "address2": "Apartment", + "city": "Kolkata", + "province": "West Bengal", + "country": "India", + "zip": "708091", + "phone": "+919876543210", + "name": "Anuraj Guha", + "province_code": "WB", + "country_code": "IN", + "country_name": "India", + "default": true + }, + "acceptsMarketing": false, + "orderCount": 0, + "state": "disabled", + "totalSpent": "0.00", + "note": "", + "verifiedEmail": true, + "taxExempt": false, + "tags": "", + "currency": "INR", + "taxExemptions": [], + "smsMarketingConsent": { + "state": "not_subscribed", + "opt_in_level": "single_opt_in", + "consent_updated_at": null, + "consent_collected_from": "SHOPIFY" + }, + "adminGraphqlApiId": "gid://shopify/Customer/5747017285820", + "acceptsMarketingUpdatedAt": "2021-12-29T15:15:20+05:30" + }, + "timestamp": "2021-12-29T09:45:20.000Z", + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "rudderId": "56ad0483-e7e8-438d-958f-676fcbf3ed49", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ] + } +} diff --git a/go/webhook/testcases/testdata/testcases/shopify/invalid_topic.json b/go/webhook/testcases/testdata/testcases/shopify/invalid_topic.json new file mode 100644 index 0000000000..f4048a0783 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/shopify/invalid_topic.json @@ -0,0 +1,24 @@ +{ + "name": "shopify", + "description": "Invalid topic", + "input": { + "request": { + "query": "signature=rudderstack" + } + }, + "output": { + "response": { + "status": 400, + "contentType": "text/plain", + "body": "Invalid topic in query_parameters\n" + }, + "err_queue": [ + { + "query_parameters": { + "writeKey": ["{{.WriteKey}}"], + "signature": ["rudderstack"] + } + } + ] + } +} diff --git a/go/webhook/testcases/testdata/testcases/shopify/no_query_params.json b/go/webhook/testcases/testdata/testcases/shopify/no_query_params.json new file mode 100644 index 0000000000..c9eda9924f --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/shopify/no_query_params.json @@ -0,0 +1,19 @@ +{ + "name": "shopify", + "description": "No Query Parameters", + "input": {}, + "output": { + "response": { + "status": 400, + "contentType": "text/plain", + "body": "Invalid topic in query_parameters\n" + }, + "err_queue": [ + { + "query_parameters": { + "writeKey": ["{{.WriteKey}}"] + } + } + ] + } +} diff --git a/go/webhook/testcases/testdata/testcases/shopify/unsupported_checkout_event.json b/go/webhook/testcases/testdata/testcases/shopify/unsupported_checkout_event.json new file mode 100644 index 0000000000..2a6f6c6642 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/shopify/unsupported_checkout_event.json @@ -0,0 +1,50 @@ +{ + "name": "shopify", + "description": "Unsupported checkout event", + "input": { + "request": { + "query": "signature=rudderstack&topic=checkout_delete", + "body": { + "admin_graphql_api_id": "gid://shopify/Fulfillment/4124667937024", + "created_at": "2022-01-05T18:13:02+05:30", + "destination": null, + "id": 4124667937024, + "line_items": [], + "customer": { + "email": "test_person@email.com", + "first_name": "Test", + "last_name": "Person" + }, + "billing_address": { + "address1": "11 Rani Sankari Lane Patuapara Bhowanipore" + }, + "shipping_address": { + "address1": "11 Rani Sankari Lane Patuapara Bhowanipore" + }, + "location_id": 66855371008, + "name": "#1002.1", + "order_id": 4617255092480, + "origin_address": null, + "receipt": {}, + "service": "manual", + "shipment_status": null, + "status": "success", + "tracking_company": "Amazon Logistics UK", + "tracking_number": "Sample001test", + "tracking_numbers": ["Sample001test"], + "tracking_url": "https://www.amazon.co.uk/gp/help/customer/display.html?nodeId=201910530", + "tracking_urls": [ + "https://www.amazon.co.uk/gp/help/customer/display.html?nodeId=201910530" + ], + "updated_at": "2022-01-05T18:16:48+05:30" + } + } + }, + "output": { + "response": { + "status": 200, + "contentType": "text/plain", + "body": "OK" + } + } +} diff --git a/go/webhook/testcases/testdata/testcases/shopify/unsupported_event_type.json b/go/webhook/testcases/testdata/testcases/shopify/unsupported_event_type.json new file mode 100644 index 0000000000..26c08e9c4f --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/shopify/unsupported_event_type.json @@ -0,0 +1,16 @@ +{ + "name": "shopify", + "description": "Unsupported event type", + "input": { + "request": { + "query": "signature=rudderstack&topic=random_event" + } + }, + "output": { + "response": { + "status": 200, + "contentType": "text/plain", + "body": "OK" + } + } +} diff --git a/go/webhook/testcases/testdata/testcases/slack/message_event.json b/go/webhook/testcases/testdata/testcases/slack/message_event.json new file mode 100644 index 0000000000..744170beec --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/slack/message_event.json @@ -0,0 +1,145 @@ +{ + "name": "slack", + "description": "Message event", + "input": { + "request": { + "body": { + "event": { + "user": "U04G7H550", + "type": "message", + "ts": "1709441309.308399", + "client_msg_id": "834r664e-ec75-445d-t5c6-b873a07y9c81", + "text": "What is the pricing of product X", + "team": "T0GFJL5J7", + "thread_ts": "1709407304.839329", + "parent_user_id": "U06P6LQTPV", + "blocks": [ + { + "type": "rich_text", + "block_id": "xGKJl", + "elements": [ + { + "type": "rich_text_section", + "elements": [ + { + "type": "text", + "text": "What is the pricing of product X" + }, + { + "type": "channel", + "channel_id": "C03CDQTPI65" + }, + { + "type": "text", + "text": " to do this" + } + ] + } + ] + } + ], + "channel": "C03CDQTPI65", + "event_ts": "1709441309.308399", + "channel_type": "channel" + }, + "type": "event_callback", + "event_id": "EvY5JTJ0NG5", + "event_time": 1709441309, + "token": "REm2987dqtpi72Lq", + "team_id": "T0GFJL5J7", + "context_team_id": "T01gqtPIL5J7", + "context_enterprise_id": null, + "api_app_id": "A04QTPIHRR", + "authorizations": [ + { + "enterprise_id": null, + "team_id": "T0GFJL5J7", + "user_id": "W012CDE", + "is_bot": true, + "is_enterprise_install": false + } + ], + "is_ext_shared_channel": false, + "event_context": "4-wd6joiQfdgTRQTpIzdfifQ" + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "SLACK" + }, + "externalId": [ + { + "type": "slackUserId", + "id": "U04G7H550" + } + ] + }, + "integrations": { + "SLACK": false + }, + "type": "track", + "event": "Message", + "anonymousId": "7509c04f547b05afb6838aa742f4910263d6", + "originalTimestamp": "2024-03-03T04:48:29.308Z", + "sentAt": "2024-03-03T04:48:29.000Z", + "properties": { + "user": "U04G7H550", + "type": "message", + "ts": "1709441309.308399", + "client_msg_id": "834r664e-ec75-445d-t5c6-b873a07y9c81", + "text": "What is the pricing of product X", + "team": "T0GFJL5J7", + "thread_ts": "1709407304.839329", + "parent_user_id": "U06P6LQTPV", + "blocks": [ + { + "type": "rich_text", + "block_id": "xGKJl", + "elements": [ + { + "type": "rich_text_section", + "elements": [ + { + "type": "text", + "text": "What is the pricing of product X" + }, + { + "type": "channel", + "channel_id": "C03CDQTPI65" + }, + { + "type": "text", + "text": " to do this" + } + ] + } + ] + } + ], + "channel": "C03CDQTPI65", + "event_ts": "1709441309.308399", + "channel_type": "channel" + }, + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/slack/msg_event.json b/go/webhook/testcases/testdata/testcases/slack/msg_event.json new file mode 100644 index 0000000000..9ca194858c --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/slack/msg_event.json @@ -0,0 +1,149 @@ +{ + "name": "slack", + "description": "Message event", + "module": "source", + "version": "v0", + "input": { + "request": { + "body": { + "event": { + "user": "U04G7H550", + "type": "message", + "ts": "1709441309.308399", + "client_msg_id": "834r664e-ec75-445d-t5c6-b873a07y9c81", + "text": "What is the pricing of product X", + "team": "T0GFJL5J7", + "thread_ts": "1709407304.839329", + "parent_user_id": "U06P6LQTPV", + "blocks": [ + { + "type": "rich_text", + "block_id": "xGKJl", + "elements": [ + { + "type": "rich_text_section", + "elements": [ + { + "type": "text", + "text": "What is the pricing of product X" + }, + { + "type": "channel", + "channel_id": "C03CDQTPI65" + }, + { + "type": "text", + "text": " to do this" + } + ] + } + ] + } + ], + "channel": "C03CDQTPI65", + "event_ts": "1709441309.308399", + "channel_type": "channel" + }, + "type": "event_callback", + "event_id": "EvY5JTJ0NG5", + "event_time": 1709441309, + "token": "REm2987dqtpi72Lq", + "team_id": "T0GFJL5J7", + "context_team_id": "T01gqtPIL5J7", + "context_enterprise_id": null, + "api_app_id": "A04QTPIHRR", + "authorizations": [ + { + "enterprise_id": null, + "team_id": "T0GFJL5J7", + "user_id": "W012CDE", + "is_bot": true, + "is_enterprise_install": false + } + ], + "is_ext_shared_channel": false, + "event_context": "4-wd6joiQfdgTRQTpIzdfifQ" + }, + "method": "POST", + "headers": { + "Content-Type": "application/json" + } + }, + "pathSuffix": "" + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "SLACK" + }, + "externalId": [ + { + "type": "slackUserId", + "id": "U04G7H550" + } + ] + }, + "integrations": { + "SLACK": false + }, + "type": "track", + "event": "Message", + "anonymousId": "7509c04f547b05afb6838aa742f4910263d6", + "originalTimestamp": "2024-03-03T04:48:29.308Z", + "sentAt": "2024-03-03T04:48:29.000Z", + "properties": { + "user": "U04G7H550", + "type": "message", + "ts": "1709441309.308399", + "client_msg_id": "834r664e-ec75-445d-t5c6-b873a07y9c81", + "text": "What is the pricing of product X", + "team": "T0GFJL5J7", + "thread_ts": "1709407304.839329", + "parent_user_id": "U06P6LQTPV", + "blocks": [ + { + "type": "rich_text", + "block_id": "xGKJl", + "elements": [ + { + "type": "rich_text_section", + "elements": [ + { + "type": "text", + "text": "What is the pricing of product X" + }, + { + "type": "channel", + "channel_id": "C03CDQTPI65" + }, + { + "type": "text", + "text": " to do this" + } + ] + } + ] + } + ], + "channel": "C03CDQTPI65", + "event_ts": "1709441309.308399", + "channel_type": "channel" + }, + "request_ip": "192.0.2.30", + "receivedAt": "2024-03-03T04:48:29.000Z", + "rudderId": "5540ba39-3df8-4970-8b50-b329182c9bd5", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ] + } +} diff --git a/go/webhook/testcases/testdata/testcases/slack/team_joined.json b/go/webhook/testcases/testdata/testcases/slack/team_joined.json new file mode 100644 index 0000000000..114872a76a --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/slack/team_joined.json @@ -0,0 +1,89 @@ +{ + "name": "slack", + "description": "Team joined event", + "module": "source", + "version": "v0", + "input": { + "request": { + "body": { + "event": { + "type": "team_join", + "user": { + "id": "W012CDE", + "name": "johnd", + "real_name": "John Doe" + } + }, + "type": "event_callback", + "event_id": "Ev06TJ0NG5", + "event_time": 1709441309, + "token": "REm276ggfh72Lq", + "team_id": "T0GFJL5J7", + "context_team_id": "T0GFJL5J7", + "context_enterprise_id": null, + "api_app_id": "B02SJMHRR", + "authorizations": [ + { + "enterprise_id": null, + "team_id": "T0GFJL5J7", + "user_id": "U04G7H550", + "is_bot": true, + "is_enterprise_install": false + } + ], + "is_ext_shared_channel": false, + "event_context": "eJldCI65436EUEpMSFhgfhg76joiQzAxRTRQTEIxMzUifQ" + }, + "method": "POST", + "headers": { + "Content-Type": "application/json" + } + }, + "pathSuffix": "" + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "SLACK" + }, + "externalId": [ + { + "type": "slackUserId", + "id": "W012CDE" + } + ] + }, + "integrations": { + "SLACK": false + }, + "type": "identify", + "event": "Team Join", + "anonymousId": "2bc5ae2825a712d3d154cbdacb86ac16c278", + "originalTimestamp": "2024-03-03T04:48:29.000Z", + "sentAt": "2024-03-03T04:48:29.000Z", + "properties": { + "type": "team_join", + "user": { + "id": "W012CDE", + "name": "johnd", + "real_name": "John Doe" + } + }, + "request_ip": "192.0.2.30", + "receivedAt": "2024-03-03T04:48:29.000Z", + "rudderId": "669b7c62-3e8b-475c-8f0a-356b2a9518e2", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ] + } +} diff --git a/go/webhook/testcases/testdata/testcases/slack/team_joined_event.json b/go/webhook/testcases/testdata/testcases/slack/team_joined_event.json new file mode 100644 index 0000000000..8b0c547f69 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/slack/team_joined_event.json @@ -0,0 +1,85 @@ +{ + "name": "slack", + "description": "Team joined event", + "input": { + "request": { + "body": { + "event": { + "type": "team_join", + "user": { + "id": "W012CDE", + "name": "johnd", + "real_name": "John Doe" + } + }, + "type": "event_callback", + "event_id": "Ev06TJ0NG5", + "event_time": 1709441309, + "token": "REm276ggfh72Lq", + "team_id": "T0GFJL5J7", + "context_team_id": "T0GFJL5J7", + "context_enterprise_id": null, + "api_app_id": "B02SJMHRR", + "authorizations": [ + { + "enterprise_id": null, + "team_id": "T0GFJL5J7", + "user_id": "U04G7H550", + "is_bot": true, + "is_enterprise_install": false + } + ], + "is_ext_shared_channel": false, + "event_context": "eJldCI65436EUEpMSFhgfhg76joiQzAxRTRQTEIxMzUifQ" + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": "OK" + }, + "queue": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "integration": { + "name": "SLACK" + }, + "externalId": [ + { + "type": "slackUserId", + "id": "W012CDE" + } + ] + }, + "integrations": { + "SLACK": false + }, + "type": "identify", + "event": "Team Join", + "anonymousId": "2bc5ae2825a712d3d154cbdacb86ac16c278", + "originalTimestamp": "2024-03-03T04:48:29.000Z", + "sentAt": "2024-03-03T04:48:29.000Z", + "properties": { + "type": "team_join", + "user": { + "id": "W012CDE", + "name": "johnd", + "real_name": "John Doe" + } + }, + "receivedAt": "2024-03-03T04:48:29.000Z", + "request_ip": "192.0.2.30", + "messageId": "00000000-0000-0000-0000-000000000000" + } + ], + "errQueue": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/slack/verification.json b/go/webhook/testcases/testdata/testcases/slack/verification.json new file mode 100644 index 0000000000..94e072dd8f --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/slack/verification.json @@ -0,0 +1,31 @@ +{ + "name": "slack", + "description": "Webhook url verification event", + "module": "source", + "version": "v0", + "input": { + "request": { + "body": { + "token": "Jhj5dZrVaK7ZwHHjRyZWjbDl", + "challenge": "3eZbrw1aB10FEMAGAZd4FyFQ", + "type": "url_verification" + }, + "method": "POST", + "headers": { + "Content-Type": "application/json" + } + }, + "pathSuffix": "" + }, + "output": { + "response": { + "status": 200, + "contentType": "application/json", + "body": { + "challenge": "3eZbrw1aB10FEMAGAZd4FyFQ" + } + }, + "queue": [], + "proc_err": [] + } +} diff --git a/go/webhook/testcases/testdata/testcases/slack/webhook_url_verificatin_event.json b/go/webhook/testcases/testdata/testcases/slack/webhook_url_verificatin_event.json new file mode 100644 index 0000000000..0be33d2c58 --- /dev/null +++ b/go/webhook/testcases/testdata/testcases/slack/webhook_url_verificatin_event.json @@ -0,0 +1,26 @@ +{ + "name": "slack", + "description": "Webhook url verificatin event", + "input": { + "request": { + "body": { + "token": "Jhj5dZrVaK7ZwHHjRyZWjbDl", + "challenge": "3eZbrw1aB10FEMAGAZd4FyFQ", + "type": "url_verification" + }, + "headers": { + "Content-Type": "application/json" + } + } + }, + "output": { + "response": { + "status": 200, + "body": { + "challenge": "3eZbrw1aB10FEMAGAZd4FyFQ" + } + }, + "queue": [], + "errQueue": [] + } +} diff --git a/test/integrations/sources/adjust/data.ts b/test/integrations/sources/adjust/data.ts index 975543fbec..733a1d6235 100644 --- a/test/integrations/sources/adjust/data.ts +++ b/test/integrations/sources/adjust/data.ts @@ -1,3 +1,4 @@ +import { skip } from 'node:test'; import utils from '../../../../src/v0/util'; const defaultMockFns = () => { @@ -10,6 +11,7 @@ export const data = [ description: 'Simple track call', module: 'source', version: 'v0', + skipGo: 'FIXME', input: { request: { body: [ @@ -85,6 +87,7 @@ export const data = [ description: 'Simple track call with no query parameters', module: 'source', version: 'v0', + skipGo: 'FIXME', input: { request: { body: [ diff --git a/test/integrations/sources/auth0/data.ts b/test/integrations/sources/auth0/data.ts index 44b511cad2..953888920b 100644 --- a/test/integrations/sources/auth0/data.ts +++ b/test/integrations/sources/auth0/data.ts @@ -10,6 +10,7 @@ export const data = [ description: 'successful signup', module: 'source', version: 'v0', + skipGo: 'dynamic anonymousId', input: { request: { body: [ @@ -532,6 +533,7 @@ export const data = [ description: 'add member to an organization', module: 'source', version: 'v0', + skipGo: 'dynamic anonymousId', input: { request: { body: [ @@ -671,6 +673,7 @@ export const data = [ description: 'update tenant settings', module: 'source', version: 'v0', + skipGo: 'dynamic anonymousId', input: { request: { body: [ @@ -1242,6 +1245,7 @@ export const data = [ description: 'missing userId', module: 'source', version: 'v0', + skipGo: 'dynamic anonymousId', input: { request: { body: [ @@ -1348,6 +1352,7 @@ export const data = [ description: 'missing userId for all the requests in a batch', module: 'source', version: 'v0', + skipGo: 'dynamic anonymousId', input: { request: { body: [ @@ -1503,6 +1508,7 @@ export const data = [ description: 'empty batch', module: 'source', version: 'v0', + skipGo: 'dynamic anonymousId', input: { request: { body: [], diff --git a/test/integrations/testTypes.ts b/test/integrations/testTypes.ts index 3df732d84f..77635f2198 100644 --- a/test/integrations/testTypes.ts +++ b/test/integrations/testTypes.ts @@ -18,6 +18,8 @@ export interface requestType { export interface responseType { status: number; + statusCode?: number; + error?: any; body?: any; headers?: Record; } @@ -40,6 +42,7 @@ export interface TestCaseData { id?: string; name: string; description: string; + skipGo?: string; scenario?: string; successCriteria?: string; comment?: string; diff --git a/test/integrations/testUtils.ts b/test/integrations/testUtils.ts index 5e0df874f8..4f5181e41e 100644 --- a/test/integrations/testUtils.ts +++ b/test/integrations/testUtils.ts @@ -24,8 +24,10 @@ const generateAlphanumericId = (size = 36) => export const getTestDataFilePaths = (dirPath: string, opts: OptionValues): string[] => { const globPattern = join(dirPath, '**', 'data.ts'); let testFilePaths = globSync(globPattern); - if (opts.destination) { - testFilePaths = testFilePaths.filter((testFile) => testFile.includes(opts.destination)); + if (opts.destination || opts.source) { + testFilePaths = testFilePaths.filter((testFile) => + testFile.includes(opts.destination || opts.source), + ); } if (opts.feature) { testFilePaths = testFilePaths.filter((testFile) => testFile.includes(opts.feature)); diff --git a/test/scripts/generateJson.ts b/test/scripts/generateJson.ts new file mode 100644 index 0000000000..7e4c3a3c0f --- /dev/null +++ b/test/scripts/generateJson.ts @@ -0,0 +1,141 @@ +import { Command, OptionValues } from 'commander'; +import path from 'path'; +import fs from 'fs'; +import { getTestData, getTestDataFilePaths, produceTestData } from '../integrations/testUtils'; +import { head } from 'lodash'; + +interface TestCaseData { + name: string; + description: string; + skip?: string; + input: Input; + output: Output; +} + +interface Input { + request: { + query: string; + body: any; + headers?: Record; + }; +} + +interface Output { + response: { + status: number; + body: any; + }; + queue: any[]; + errQueue: any[]; +} + +const jsonGenerator = new Command(); +jsonGenerator + .name('json-generator') + .description('CLI to some JavaScript string utilities') + .version('0.8.0'); + +jsonGenerator + .command('sources') + .description('generator JSON test cases for source') + .argument('', 'output path') + .option('-s, --source ', 'source', ',') + .action(generateSources); + +jsonGenerator.parse(); + +function generateSources(outputFolder: string, options: OptionValues) { + const rootDir = __dirname; + const resolvedpath = path.resolve(rootDir, '../integrations/sources'); + + const files = getTestDataFilePaths(resolvedpath, options); + + files.forEach((testDataPath) => { + let testData = getTestData(testDataPath); + testData.forEach((testCase) => { + let statusCode: number = + testCase.output.response?.statusCode || testCase.output.response?.status || 200; + + let responseBody: any = 'OK'; + if (statusCode == 200) { + if (testCase.output.response?.body[0]?.outputToSource?.body) { + responseBody = JSON.parse( + Buffer.from(testCase.output.response?.body[0]?.outputToSource?.body, 'base64').toString( + 'utf-8', + ), + ); + } + } else { + responseBody = testCase.output.response?.error; + } + + testCase.input.request.body.forEach((body) => { + delete body['receivedAt']; + delete body['request_ip']; + }); + + let goTest: TestCaseData = { + name: testCase.name, + description: testCase.description, + input: { + request: { + query: JSON.stringify(testCase.input.request.params), + body: + testCase.input.request.body.length === 1 + ? testCase.input.request.body[0] + : testCase.input.request.body, + headers: testCase.input.request.headers || { + 'Content-Type': 'application/json', + }, + }, + }, + output: { + response: { + status: statusCode, + body: responseBody, + }, + // TODO flatten nested array + queue: + statusCode == 200 + ? testCase.output.response?.body + .filter((i) => i.output) + .map((i) => i.output.batch) + .flat() + : [], + errQueue: statusCode != 200 ? [testCase.output.response?.body] : [], + }, + }; + const dirPath = path.join(outputFolder, goTest.name); + const filePath = path.join(dirPath, `${toSnakeCase(goTest.description)}.json`); + + if (testCase.skipGo) { + goTest.skip = testCase.skipGo; + } + + goTest.output.queue.forEach((queueItem) => { + queueItem['receivedAt'] = '2024-03-03T04:48:29.000Z'; + queueItem['request_ip'] = '192.0.2.30'; + if (!queueItem['messageId']) { + queueItem['messageId'] = '00000000-0000-0000-0000-000000000000'; + } + }); + + fs.mkdirSync(dirPath, { recursive: true }); + + fs.writeFileSync(filePath, JSON.stringify(goTest, null, 2)); + }); + }); +} + +function toSnakeCase(str: string): string { + return ( + str + // Replace spaces with underscores + .replace(/\s+/g, '_') + // Insert underscores before uppercase letters, handle acronyms correctly + .replace(/\.?([A-Z]+)/g, (x, y) => '_' + y.toLowerCase()) + // Remove leading underscores and handle consecutive underscores + .replace(/^_+/, '') + .replace(/_{2,}/g, '_') + ); +} diff --git a/tsconfig.json b/tsconfig.json index 926831b612..2c00e6482e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -102,6 +102,6 @@ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */, }, - "exclude": ["./src/**/*.test.js", "./src/**/*.test.ts", "./test"], + "exclude": ["./src/**/*.test.js", "./src/**/*.test.ts", "./test", "./go"], "include": ["./src", "./src/**/*.json"], } From 68367f5227c96f2700a773018b991b1e87a0774d Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Wed, 24 Jul 2024 11:50:25 +0530 Subject: [PATCH 15/28] fix: update getConversionActionId function for gaoc (#3594) --- .../utils.js | 5 +- .../dataDelivery/oauth.ts | 72 +++++++++++++++++++ .../network.ts | 69 +++++++++++------- 3 files changed, 119 insertions(+), 27 deletions(-) diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js index 7c21371ccf..7228bb8da4 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.js @@ -16,7 +16,6 @@ const { getFieldValueFromMessage, isDefinedAndNotNullAndNotEmpty, isDefinedAndNotNull, - getAuthErrCategoryFromStCode, getAccessToken, getIntegrationsObj, } = require('../../util'); @@ -34,7 +33,7 @@ const { const { processAxiosResponse } = require('../../../adapters/utils/networkUtils'); const Cache = require('../../util/cache'); const helper = require('./helper'); -const { finaliseConsent } = require('../../util/googleUtils'); +const { finaliseConsent, getAuthErrCategory } = require('../../util/googleUtils'); const conversionActionIdCache = new Cache(CONVERSION_ACTION_ID_CACHE_TTL); @@ -86,7 +85,7 @@ const getConversionActionId = async ({ headers, params, metadata }) => { )} during google_ads_offline_conversions response transformation`, status, response, - getAuthErrCategoryFromStCode(get(searchStreamResponse, 'status')), + getAuthErrCategory(searchStreamResponse), ); } const conversionAction = get( diff --git a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts index 73fd7c961d..b8e4c78e1d 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts @@ -332,4 +332,76 @@ export const v1oauthScenarios = [ }, }, }, + { + id: 'gaoc_v1_oauth_scenario_4', + name: 'google_adwords_offline_conversions', + description: + "[Proxy v1 API] :: Oauth when the user doesn't enabled 2 factor authentication but the google ads account has it enabled for not store sales conversion", + successCriteria: 'The proxy should return 401 with authErrorCategory as AUTH_STATUS_INACTIVE', + scenario: 'Oauth', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + ...{ ...commonRequestParameters, JSON: { isStoreConversion: false } }, + headers: { + Authorization: 'Bearer invalidabcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + 'login-customer-id': 'logincustomerid', + }, + endpoint: + 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 401, + body: { + output: { + authErrorCategory: 'AUTH_STATUS_INACTIVE', + message: + '[Google Ads Offline Conversions]:: {"error":{"code":401,"details":[{"@type":"type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure","errors":[{"errorCode":{"authenticationError":"TWO_STEP_VERIFICATION_NOT_ENROLLED"},"message":"An account administrator changed this account\'s authentication settings. To access this Google Ads account, enable 2-Step Verification in your Google account at https://www.google.com/landing/2step."}],"requestId":"wy4ZYbsjWcgh6uC2Ruc_Zg"}],"message":"Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.","status":"UNAUTHENTICATED"}} during google_ads_offline_conversions response transformation', + response: [ + { + error: + '[Google Ads Offline Conversions]:: {"error":{"code":401,"details":[{"@type":"type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure","errors":[{"errorCode":{"authenticationError":"TWO_STEP_VERIFICATION_NOT_ENROLLED"},"message":"An account administrator changed this account\'s authentication settings. To access this Google Ads account, enable 2-Step Verification in your Google account at https://www.google.com/landing/2step."}],"requestId":"wy4ZYbsjWcgh6uC2Ruc_Zg"}],"message":"Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.","status":"UNAUTHENTICATED"}} during google_ads_offline_conversions response transformation', + metadata: { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + jobId: 1, + secret: { + accessToken: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + }, + statusCode: 401, + }, + ], + statTags: { + destType: 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + status: 401, + }, + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/google_adwords_offline_conversions/network.ts b/test/integrations/destinations/google_adwords_offline_conversions/network.ts index 49f89cec28..f1147162d2 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/network.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/network.ts @@ -1,3 +1,30 @@ +const commonResponse = { + status: 401, + data: { + error: { + code: 401, + details: [ + { + '@type': 'type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure', + errors: [ + { + errorCode: { + authenticationError: 'TWO_STEP_VERIFICATION_NOT_ENROLLED', + }, + message: + "An account administrator changed this account's authentication settings. To access this Google Ads account, enable 2-Step Verification in your Google account at https://www.google.com/landing/2step.", + }, + ], + requestId: 'wy4ZYbsjWcgh6uC2Ruc_Zg', + }, + ], + message: + 'Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.', + status: 'UNAUTHENTICATED', + }, + }, +}; + export const networkCallsData = [ { httpReq: { @@ -647,7 +674,7 @@ export const networkCallsData = [ }, { description: - 'Mock response from destination depicting a request with invalid authentication credentials', + 'Mock response from destination depicting a request from user who has not enabled 2 factor authentication', httpReq: { url: 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs:create', data: { @@ -669,31 +696,25 @@ export const networkCallsData = [ }, method: 'POST', }, - httpRes: { - status: 401, + httpRes: commonResponse, + }, + { + description: + 'Mock response from destination depicting a request from user who has not enabled 2 factor authentication', + httpReq: { + url: 'https://googleads.googleapis.com/v16/customers/1112223333/googleAds:searchStream', data: { - error: { - code: 401, - details: [ - { - '@type': 'type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure', - errors: [ - { - errorCode: { - authenticationError: 'TWO_STEP_VERIFICATION_NOT_ENROLLED', - }, - message: - "An account administrator changed this account's authentication settings. To access this Google Ads account, enable 2-Step Verification in your Google account at https://www.google.com/landing/2step.", - }, - ], - requestId: 'wy4ZYbsjWcgh6uC2Ruc_Zg', - }, - ], - message: - 'Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.', - status: 'UNAUTHENTICATED', - }, + query: + "SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Sign-up - click'", + }, + headers: { + Authorization: 'Bearer invalidabcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + 'login-customer-id': 'logincustomerid', }, + method: 'POST', }, + httpRes: commonResponse, }, ]; From 3f75b55daf2a0215592df458c1f3e20f8ff40b0c Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Wed, 24 Jul 2024 12:19:35 +0530 Subject: [PATCH 16/28] chore: enable labels for api calls (#3476) * chore: enable labels for api calls * chore: propagate metadata to api call fn in intercom dest * chore: metadata propagation in mautic & optimizely_fullstack Signed-off-by: Sai Sankeerth * chore: metadata propagation in active_campaign Signed-off-by: Sai Sankeerth * chore: metadata propagation in delighted & klaviyo Signed-off-by: Sai Sankeerth * chore: add method doc to getRelativePathFromURL * chore: add test-cases to test firing of stats based on metadata * chore: enable labels for api calls - 2 (#3479) * chore: enable labels for api call - 2 * chore: propagate metadata in following destinations - canny - clickup - custify - freshmarketer - freshsales - wootric * chore: propagate metadata for below destinations - monday - profitwell - sendgrid - sendinblue * chore: fix sfmc transformation test * chore: enable labels for api calls - 3 (#3483) * chore: enable labels for api calls - 3 * chore: propagate metadata for user destination * chore: propagate metadata in braze * chore: propagate metadata for api calls in hubspot * chore: propagate metadata for labels in api calls for - pardot - rakuten - snapchat_custom_audience - the_trade_desk - yahoo_dsp * chore: propagate metadata in zendesk Signed-off-by: Sai Sankeerth * chore: propagate metadata in gainsight --------- Signed-off-by: Sai Sankeerth Co-authored-by: Sai Sankeerth * chore: update canny transformation to include await keyword Co-authored-by: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> * chore: add async for canny destination - fix eslint error --------- Signed-off-by: Sai Sankeerth Co-authored-by: Sai Sankeerth Co-authored-by: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> * chore: remove duplicate metadata key in statTags obj klaviyo Signed-off-by: Sai Sankeerth --------- Signed-off-by: Sai Sankeerth Co-authored-by: Sai Sankeerth Co-authored-by: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> --- src/adapters/network.js | 18 +- src/adapters/network.test.js | 313 +++++++++++++++++- .../destinations/intercom/procWorkflow.yaml | 20 +- src/cdk/v2/destinations/intercom/utils.js | 12 +- .../v2/destinations/intercom/utils.test.js | 16 +- .../optimizely_fullstack/procWorkflow.yaml | 6 +- .../destinations/active_campaign/transform.js | 45 ++- src/v0/destinations/braze/braze.util.test.js | 10 +- src/v0/destinations/braze/transform.js | 5 +- src/v0/destinations/braze/util.js | 7 +- src/v0/destinations/canny/transform.js | 16 +- src/v0/destinations/canny/util.js | 3 +- src/v0/destinations/clickup/transform.js | 9 +- src/v0/destinations/clickup/util.js | 13 +- src/v0/destinations/custify/transform.js | 10 +- src/v0/destinations/custify/util.js | 7 +- src/v0/destinations/delighted/transform.js | 8 +- src/v0/destinations/delighted/util.js | 3 +- .../destinations/freshmarketer/transform.js | 26 +- src/v0/destinations/freshmarketer/utils.js | 32 +- src/v0/destinations/freshsales/transform.js | 23 +- src/v0/destinations/freshsales/utils.js | 31 +- src/v0/destinations/hs/HSTransform-v1.js | 13 +- src/v0/destinations/hs/HSTransform-v2.js | 10 +- src/v0/destinations/hs/transform.js | 31 +- src/v0/destinations/hs/util.js | 25 +- src/v0/destinations/klaviyo/util.js | 2 +- src/v0/destinations/marketo/transform.js | 54 +-- src/v0/destinations/marketo/util.js | 6 +- .../marketo_static_list/transform.js | 6 +- src/v0/destinations/mautic/transform.js | 20 +- src/v0/destinations/mautic/utils.js | 3 +- src/v0/destinations/monday/transform.js | 10 +- src/v0/destinations/monday/util.js | 3 +- src/v0/destinations/pardot/networkHandler.js | 2 + src/v0/destinations/profitwell/transform.js | 13 +- src/v0/destinations/profitwell/utils.js | 3 +- src/v0/destinations/rakuten/networkHandler.js | 2 + src/v0/destinations/salesforce/transform.js | 40 ++- src/v0/destinations/salesforce/utils.js | 7 +- src/v0/destinations/sendgrid/transform.js | 15 +- src/v0/destinations/sendgrid/util.js | 11 +- src/v0/destinations/sendinblue/transform.js | 15 +- src/v0/destinations/sendinblue/util.js | 3 +- src/v0/destinations/sfmc/transform.js | 13 +- src/v0/destinations/sfmc/transform.test.js | 10 +- .../networkHandler.js | 2 + .../the_trade_desk/networkHandler.js | 2 + src/v0/destinations/user/transform.js | 40 +-- src/v0/destinations/user/utils.js | 35 +- src/v0/destinations/wootric/transform.js | 12 +- src/v0/destinations/wootric/util.js | 6 +- src/v0/destinations/yahoo_dsp/transform.js | 10 +- src/v0/destinations/yahoo_dsp/util.js | 3 +- src/v0/destinations/zendesk/transform.js | 92 +++-- src/v0/util/index.js | 17 + src/v0/util/index.test.js | 21 ++ 57 files changed, 872 insertions(+), 318 deletions(-) diff --git a/src/adapters/network.js b/src/adapters/network.js index 86af19d6a1..aeb1cc128b 100644 --- a/src/adapters/network.js +++ b/src/adapters/network.js @@ -5,9 +5,14 @@ const lodash = require('lodash'); const http = require('http'); const https = require('https'); const axios = require('axios'); -const logger = require('../logger'); +const { isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); const stats = require('../util/stats'); -const { removeUndefinedValues, isDefinedAndNotNullAndNotEmpty } = require('../v0/util'); +const { + removeUndefinedValues, + getErrorStatusCode, + isDefinedAndNotNullAndNotEmpty, +} = require('../v0/util'); +const logger = require('../logger'); const { processAxiosResponse } = require('./utils/networkUtils'); // Only for tests const { setResponsesForMockAxiosAdapter } = require('../../test/testHelper'); @@ -83,7 +88,9 @@ const fireHTTPStats = (clientResponse, startTime, statTags) => { const endpointPath = statTags.endpointPath ? statTags.endpointPath : ''; const requestMethod = statTags.requestMethod ? statTags.requestMethod : ''; const module = statTags.module ? statTags.module : ''; - const statusCode = clientResponse.success ? clientResponse.response.status : ''; + const statusCode = clientResponse.success + ? clientResponse.response.status + : getErrorStatusCode(clientResponse.response); const defArgs = { destType, endpointPath, @@ -94,9 +101,9 @@ const fireHTTPStats = (clientResponse, startTime, statTags) => { startTime, clientResponse, }; - if (statTags?.metadata) { + if (statTags?.metadata && typeof statTags?.metadata === 'object') { const metadata = !Array.isArray(statTags?.metadata) ? [statTags.metadata] : statTags.metadata; - metadata?.forEach((m) => { + metadata?.filter(isDefinedAndNotNull)?.forEach((m) => { fireOutgoingReqStats({ ...defArgs, metadata: m, @@ -448,4 +455,5 @@ module.exports = { getFormData, handleHttpRequest, enhanceRequestOptions, + fireHTTPStats, }; diff --git a/src/adapters/network.test.js b/src/adapters/network.test.js index 5f5ad97437..7894925ccd 100644 --- a/src/adapters/network.test.js +++ b/src/adapters/network.test.js @@ -1,8 +1,17 @@ const mockLoggerInstance = { info: jest.fn(), }; -const { getFormData, httpPOST, httpGET, httpSend } = require('./network'); +const { getFormData, httpPOST, httpGET, httpSend, fireHTTPStats } = require('./network'); const { getFuncTestData } = require('../../test/testHelper'); +jest.mock('../util/stats', () => ({ + timing: jest.fn(), + timingSummary: jest.fn(), + increment: jest.fn(), + counter: jest.fn(), + gauge: jest.fn(), + histogram: jest.fn(), +})); +const stats = require('../util/stats'); jest.mock('@rudderstack/integrations-lib', () => { return { @@ -39,6 +48,308 @@ describe(`${funcName} Tests`, () => { }); }); +describe('fireHTTPStats tests', () => { + beforeEach(() => { + stats.timing.mockClear(); + stats.counter.mockClear(); + }); + it('should not throw error when metadata is sent as correctly defined object', () => { + const clientResponse = { + success: true, + response: { + data: { a: 1, b: 2 }, + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + }; + const startTime = new Date(); + const statTags = { + metadata: { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + }, + destType: 'DT', + feature: 'feat', + endpointPath: '/some/url', + requestMethod: 'post', + }; + expect(() => { + fireHTTPStats(clientResponse, startTime, statTags); + }).not.toThrow(Error); + expect(stats.timing).toHaveBeenCalledTimes(1); + expect(stats.timing).toHaveBeenNthCalledWith(1, 'outgoing_request_latency', startTime, { + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + destType: 'DT', + feature: 'feat', + module: '', + endpointPath: '/some/url', + requestMethod: 'post', + }); + }); + it('should not throw error when metadata is sent as correctly defined object[]', () => { + const clientResponse = { + success: false, + response: { + response: { + data: { errors: [{ e: 'something went bad' }] }, + status: 400, + headers: { 'Content-Type': 'application/json' }, + }, + headers: { 'Content-Type': 'application/json' }, + status: 400, + }, + }; + const startTime = new Date(); + const statTags = { + metadata: [ + { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + jobId: 1, + sourceId: 's1', + }, + { + destType: 'DT', + jobId: 2, + destinationId: 'd1', + workspaceId: 'w2', + sourceId: 's2', + }, + ], + destType: 'DT', + feature: 'feat', + endpointPath: '/some/url', + requestMethod: 'post', + }; + expect(() => { + fireHTTPStats(clientResponse, startTime, statTags); + }).not.toThrow(Error); + + expect(stats.timing).toHaveBeenCalledTimes(2); + expect(stats.timing).toHaveBeenNthCalledWith(1, 'outgoing_request_latency', startTime, { + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + destType: 'DT', + feature: 'feat', + module: '', + endpointPath: '/some/url', + requestMethod: 'post', + }); + expect(stats.timing).toHaveBeenNthCalledWith(2, 'outgoing_request_latency', startTime, { + destinationId: 'd1', + workspaceId: 'w2', + sourceId: 's2', + destType: 'DT', + module: '', + feature: 'feat', + endpointPath: '/some/url', + requestMethod: 'post', + }); + }); + it('should not throw error when metadata is not sent', () => { + const clientResponse = { + success: false, + response: { + response: { + data: { errors: [{ e: 'something went bad' }] }, + status: 400, + headers: { 'Content-Type': 'application/json' }, + }, + headers: { 'Content-Type': 'application/json' }, + status: 400, + }, + }; + const startTime = new Date(); + const statTags = { + destType: 'DT', + feature: 'feat', + endpointPath: '/some/url', + requestMethod: 'post', + }; + expect(() => { + fireHTTPStats(clientResponse, startTime, statTags); + }).not.toThrow(Error); + + expect(stats.timing).toHaveBeenCalledTimes(1); + expect(stats.timing).toHaveBeenNthCalledWith(1, 'outgoing_request_latency', startTime, { + destType: 'DT', + feature: 'feat', + module: '', + endpointPath: '/some/url', + requestMethod: 'post', + }); + }); + it('should not throw error when metadata is sent as empty array', () => { + const clientResponse = { + success: false, + response: { + response: { + data: { errors: [{ e: 'something went bad' }] }, + status: 400, + headers: { 'Content-Type': 'application/json' }, + }, + headers: { 'Content-Type': 'application/json' }, + status: 400, + }, + }; + const startTime = new Date(); + const statTags = { + metadata: [], + destType: 'DT', + feature: 'feat', + endpointPath: '/some/url', + requestMethod: 'post', + }; + expect(() => { + fireHTTPStats(clientResponse, startTime, statTags); + }).not.toThrow(Error); + + expect(stats.timing).toHaveBeenCalledTimes(0); + }); + it('should not throw error when metadata is sent as empty object', () => { + const clientResponse = { + success: false, + response: { + response: { + data: { errors: [{ e: 'something went bad' }] }, + status: 400, + headers: { 'Content-Type': 'application/json' }, + }, + headers: { 'Content-Type': 'application/json' }, + status: 400, + }, + }; + const startTime = new Date(); + const statTags = { + metadata: {}, + destType: 'DT', + feature: 'feat', + endpointPath: '/some/url', + requestMethod: 'post', + }; + expect(() => { + fireHTTPStats(clientResponse, startTime, statTags); + }).not.toThrow(Error); + + expect(stats.timing).toHaveBeenCalledTimes(1); + expect(stats.timing).toHaveBeenNthCalledWith(1, 'outgoing_request_latency', startTime, { + destType: 'DT', + feature: 'feat', + module: '', + endpointPath: '/some/url', + requestMethod: 'post', + }); + }); + it('should not throw error when metadata is sent as [null, null]', () => { + const clientResponse = { + success: false, + response: { + response: { + data: { errors: [{ e: 'something went bad' }] }, + status: 400, + headers: { 'Content-Type': 'application/json' }, + }, + headers: { 'Content-Type': 'application/json' }, + status: 400, + }, + }; + const startTime = new Date(); + const statTags = { + metadata: [null, null], + destType: 'DT', + feature: 'feat', + endpointPath: '/some/url', + requestMethod: 'post', + }; + expect(() => { + fireHTTPStats(clientResponse, startTime, statTags); + }).not.toThrow(Error); + + expect(stats.timing).toHaveBeenCalledTimes(0); + }); + it('should not throw error when metadata is sent as [1, 2]', () => { + const clientResponse = { + success: false, + response: { + response: { + data: { errors: [{ e: 'something went bad' }] }, + status: 400, + headers: { 'Content-Type': 'application/json' }, + }, + headers: { 'Content-Type': 'application/json' }, + status: 400, + }, + }; + const startTime = new Date(); + const statTags = { + metadata: [1, 2], + destType: 'DT', + feature: 'feat', + endpointPath: '/some/url', + requestMethod: 'post', + }; + expect(() => { + fireHTTPStats(clientResponse, startTime, statTags); + }).not.toThrow(Error); + + expect(stats.timing).toHaveBeenCalledTimes(2); + expect(stats.timing).toHaveBeenNthCalledWith(1, 'outgoing_request_latency', startTime, { + destType: 'DT', + feature: 'feat', + module: '', + endpointPath: '/some/url', + requestMethod: 'post', + }); + expect(stats.timing).toHaveBeenNthCalledWith(2, 'outgoing_request_latency', startTime, { + destType: 'DT', + feature: 'feat', + module: '', + endpointPath: '/some/url', + requestMethod: 'post', + }); + }); + it('should not throw error when metadata is sent as 1', () => { + const clientResponse = { + success: false, + response: { + response: { + data: { errors: [{ e: 'something went bad' }] }, + status: 400, + headers: { 'Content-Type': 'application/json' }, + }, + headers: { 'Content-Type': 'application/json' }, + status: 400, + }, + }; + const startTime = new Date(); + const statTags = { + metadata: 1, + destType: 'DT', + feature: 'feat', + endpointPath: '/some/url', + requestMethod: 'post', + }; + expect(() => { + fireHTTPStats(clientResponse, startTime, statTags); + }).not.toThrow(Error); + + expect(stats.timing).toHaveBeenCalledTimes(1); + expect(stats.timing).toHaveBeenNthCalledWith(1, 'outgoing_request_latency', startTime, { + destType: 'DT', + feature: 'feat', + module: '', + endpointPath: '/some/url', + requestMethod: 'post', + }); + }); +}); + describe('logging in http methods', () => { beforeEach(() => { mockLoggerInstance.info.mockClear(); diff --git a/src/cdk/v2/destinations/intercom/procWorkflow.yaml b/src/cdk/v2/destinations/intercom/procWorkflow.yaml index 008b22ace5..d826716b43 100644 --- a/src/cdk/v2/destinations/intercom/procWorkflow.yaml +++ b/src/cdk/v2/destinations/intercom/procWorkflow.yaml @@ -52,7 +52,7 @@ steps: - name: searchContact condition: ($.outputs.messageType === {{$.EventType.IDENTIFY}} || $.outputs.messageType === {{$.EventType.GROUP}}) && $.outputs.apiVersion !== "v1" template: | - const contactId = await $.searchContact(.message, .destination); + const contactId = await $.searchContact(.message, .destination, .metadata); contactId; - name: identifyTransformationForLatestVersion @@ -178,7 +178,7 @@ steps: condition: $.isDefinedAndNotNull($.outputs.searchContact) template: | const contactId = $.outputs.searchContact; - const companyId = await $.createOrUpdateCompany($.context.payload, .destination); + const companyId = await $.createOrUpdateCompany($.context.payload, .destination, .metadata); $.assert(companyId, "Unable to create or update company"); $.context.payload = { id: companyId, @@ -186,15 +186,16 @@ steps: $.context.endpoint = $.getBaseEndpoint(.destination) + "/" + "contacts" + "/" + contactId + "/" + "companies"; const payload = $.context.payload; const endpoint = $.context.endpoint; - await $.attachContactToCompany(payload, endpoint, .destination); - await $.addOrUpdateTagsToCompany(.message, .destination, companyId); + const eventData = {metadata: .metadata, destination: .destination} + await $.attachContactToCompany(payload, endpoint, eventData); + await $.addOrUpdateTagsToCompany({metadata: .metadata, destination: .destination, message: .message}, companyId); else: name: whenSearchContactNotFound template: | - const companyId = await $.createOrUpdateCompany($.context.payload, .destination); + const companyId = await $.createOrUpdateCompany($.context.payload, .destination, .metadata); $.assert(companyId, "Unable to create or update company"); $.context.endpoint = $.getBaseEndpoint(.destination) + "/" + "companies"; - await $.addOrUpdateTagsToCompany(.message, .destination, companyId); + await $.addOrUpdateTagsToCompany({metadata: .metadata, destination: .destination, message: .message}, companyId); - name: prepareFinalPayload template: | $.context.requestMethod = 'POST'; @@ -218,15 +219,16 @@ steps: response.userId = .message.anonymousId; $.context.response.push(response); payload = response.body.JSON; - const companyId = await $.createOrUpdateCompany(payload, .destination); + const companyId = await $.createOrUpdateCompany(payload, .destination, .metadata); $.assert(companyId, "Unable to create or update company"); const attachUserAndCompanyResponse = $.attachUserAndCompany(.message, .destination.Config); attachUserAndCompanyResponse ? attachUserAndCompanyResponse.userId = .message.anonymousId; attachUserAndCompanyResponse ? $.context.response.push(attachUserAndCompanyResponse); payload = attachUserAndCompanyResponse.body.JSON; let endpoint = attachUserAndCompanyResponse.endpoint; - attachUserAndCompanyResponse ? await $.attachContactToCompany(payload, endpoint, .destination); - await $.addOrUpdateTagsToCompany(.message, .destination, companyId); + const eventData = {metadata: .metadata, destination: .destination} + attachUserAndCompanyResponse ? await $.attachContactToCompany(payload, endpoint, eventData); + await $.addOrUpdateTagsToCompany({metadata: .metadata, destination: .destination, message: .message}, companyId); - name: statusCode condition: $.outputs.messageType === {{$.EventType.GROUP}} diff --git a/src/cdk/v2/destinations/intercom/utils.js b/src/cdk/v2/destinations/intercom/utils.js index baef94e97c..e2d8a36874 100644 --- a/src/cdk/v2/destinations/intercom/utils.js +++ b/src/cdk/v2/destinations/intercom/utils.js @@ -267,7 +267,7 @@ const filterCustomAttributes = (payload, type, destination) => { * @param {*} destination * @returns */ -const searchContact = async (message, destination) => { +const searchContact = async (message, destination, metadata) => { const lookupField = getLookUpField(message); const lookupFieldValue = getFieldValueFromMessage(message, lookupField); const data = JSON.stringify({ @@ -298,6 +298,7 @@ const searchContact = async (message, destination) => { endpointPath: '/contacts/search', requestMethod: 'POST', module: 'router', + metadata, }, ); const processedUserResponse = processAxiosResponse(response); @@ -324,7 +325,7 @@ const searchContact = async (message, destination) => { * @param {*} destination * @returns */ -const createOrUpdateCompany = async (payload, destination) => { +const createOrUpdateCompany = async (payload, destination, metadata) => { const { apiVersion } = destination.Config; const headers = getHeaders(destination, apiVersion); const finalPayload = JSON.stringify(removeUndefinedAndNullValues(payload)); @@ -337,6 +338,7 @@ const createOrUpdateCompany = async (payload, destination) => { headers, }, { + metadata, destType: 'intercom', feature: 'transformation', endpointPath: '/companies', @@ -418,7 +420,7 @@ const addMetadataToPayload = (payload) => { * @param {*} endpoint * @param {*} destination */ -const attachContactToCompany = async (payload, endpoint, destination) => { +const attachContactToCompany = async (payload, endpoint, { destination, metadata }) => { let { apiVersion } = destination.Config; apiVersion = isDefinedAndNotNull(apiVersion) ? apiVersion : 'v2'; let endpointPath = '/contact/{id}/companies'; @@ -442,6 +444,7 @@ const attachContactToCompany = async (payload, endpoint, destination) => { { ...commonStatTags, endpointPath, + metadata, }, ); @@ -460,7 +463,7 @@ const attachContactToCompany = async (payload, endpoint, destination) => { * @param id * @returns */ -const addOrUpdateTagsToCompany = async (message, destination, id) => { +const addOrUpdateTagsToCompany = async ({ message, destination, metadata }, id) => { const companyTags = message?.context?.traits?.tags; if (!companyTags) return; const { apiVersion } = destination.Config; @@ -473,6 +476,7 @@ const addOrUpdateTagsToCompany = async (message, destination, id) => { endpointPath: '/tags', requestMethod: 'POST', module: 'router', + metadata, }; await Promise.all( companyTags.map(async (tag) => { diff --git a/src/cdk/v2/destinations/intercom/utils.test.js b/src/cdk/v2/destinations/intercom/utils.test.js index bb5ad2b454..c2bf3f8e89 100644 --- a/src/cdk/v2/destinations/intercom/utils.test.js +++ b/src/cdk/v2/destinations/intercom/utils.test.js @@ -785,7 +785,7 @@ describe('attachContactToCompany utility test', () => { }, }); - await attachContactToCompany(payload, endpoint, destination); + await attachContactToCompany(payload, endpoint, { destination }); expect(axios.post).toHaveBeenCalledWith( endpoint, @@ -828,7 +828,7 @@ describe('attachContactToCompany utility test', () => { }, }); - await attachContactToCompany(payload, endpoint, destination); + await attachContactToCompany(payload, endpoint, { destination }); expect(axios.post).toHaveBeenCalledWith( endpoint, @@ -863,7 +863,7 @@ describe('attachContactToCompany utility test', () => { }); try { - await attachContactToCompany(payload, endpoint, destination); + await attachContactToCompany(payload, endpoint, { destination }); } catch (error) { expect(error.message).toEqual( 'Unable to attach Contact or User to Company due to : {"type":"error.list","request_id":"123","errors":[{"code":"company_not_found","message":"Company Not Found"}]}', @@ -893,7 +893,7 @@ describe('attachContactToCompany utility test', () => { }); try { - await attachContactToCompany(payload, endpoint, destination); + await attachContactToCompany(payload, endpoint, { destination }); } catch (error) { expect(error.message).toEqual( 'Unable to attach Contact or User to Company due to : {"type":"error.list","request_id":"123","errors":[{"code":"parameter_not_found","message":"company not specified"}]}', @@ -925,7 +925,7 @@ describe('addOrUpdateTagsToCompany utility test', () => { }); axios.post.mockClear(); - await addOrUpdateTagsToCompany(message, destination, id); + await addOrUpdateTagsToCompany({ message, destination }, id); expect(axios.post).toHaveBeenCalledTimes(2); @@ -973,7 +973,7 @@ describe('addOrUpdateTagsToCompany utility test', () => { try { axios.post.mockClear(); - await addOrUpdateTagsToCompany(message, destination, id); + await addOrUpdateTagsToCompany({ message, destination }, id); } catch (error) { expect(error.message).toEqual( `Unable to Add or Update the Tag to Company due to : {"type":"error.list","request_id":"request_401","errors":[{"code":"unauthorized","message":"Access Token Invalid"}]}`, @@ -1008,7 +1008,7 @@ describe('addOrUpdateTagsToCompany utility test', () => { try { axios.post.mockClear(); - await addOrUpdateTagsToCompany(message, destination, id); + await addOrUpdateTagsToCompany({ message, destination }, id); } catch (error) { expect(error.message).toEqual( `Unable to Add or Update the Tag to Company due to : {"type":"error.list","request_id":"request_429","errors":[{"code":"rate_limit_exceeded","message":"You have exceeded the rate limit. Please try again later."}]}`, @@ -1022,7 +1022,7 @@ describe('addOrUpdateTagsToCompany utility test', () => { const id = 'companyId'; axios.post.mockClear(); - await addOrUpdateTagsToCompany(message, destination, id); + await addOrUpdateTagsToCompany({ message, destination }, id); expect(axios.post).not.toHaveBeenCalled(); }); diff --git a/src/cdk/v2/destinations/optimizely_fullstack/procWorkflow.yaml b/src/cdk/v2/destinations/optimizely_fullstack/procWorkflow.yaml index 4d90065f7e..0776e7e8a8 100644 --- a/src/cdk/v2/destinations/optimizely_fullstack/procWorkflow.yaml +++ b/src/cdk/v2/destinations/optimizely_fullstack/procWorkflow.yaml @@ -17,6 +17,8 @@ bindings: path: ../../../../v0/util - name: generateUUID path: ../../../../v0/util + - name: getRelativePathFromURL + path: ../../../../v0/util - name: handleHttpRequest path: ../../../../adapters/network - path: ./utils @@ -59,7 +61,9 @@ steps: description: Fetch the data file from the data file url template: | const dataFileUrl = .destination.Config.dataFileUrl; - const rawResponse = await $.handleHttpRequest("get", dataFileUrl); + const urlPath = $.getRelativePathFromURL(dataFileUrl) + const reqStats = {metadata:.metadata, module: 'router',feature: "transformation", destType:"optimizely_fullstack",requestMethod:"get",endpointPath: urlPath} + const rawResponse = await $.handleHttpRequest("get", dataFileUrl, reqStats); const processedResponse = rawResponse.processedResponse; $.assertHttpResp(processedResponse, "Data File Lookup Failed due to " + JSON.stringify(processedResponse.response)); processedResponse.response; diff --git a/src/v0/destinations/active_campaign/transform.js b/src/v0/destinations/active_campaign/transform.js index 3978f868b1..f21bb1a70d 100644 --- a/src/v0/destinations/active_campaign/transform.js +++ b/src/v0/destinations/active_campaign/transform.js @@ -49,7 +49,7 @@ const responseBuilderSimple = (payload, category, destination) => { throw new TransformationError('Payload could not be constructed'); }; -const syncContact = async (contactPayload, category, destination) => { +const syncContact = async (contactPayload, category, { destination, metadata }) => { const { endPoint } = category; const endpoint = `${destination.Config.apiUrl}${endPoint}`; const requestData = { @@ -60,6 +60,7 @@ const syncContact = async (contactPayload, category, destination) => { }; const res = await httpPOST(endpoint, requestData, requestOptions, { destType: 'active_campaign', + metadata, feature: 'transformation', endpointPath: endPoint, requestMethod: 'POST', @@ -75,7 +76,7 @@ const syncContact = async (contactPayload, category, destination) => { return createdContact.id; }; -const customTagProcessor = async (message, category, destination, contactId) => { +const customTagProcessor = async ({ message, destination, metadata }, category, contactId) => { const tagsToBeCreated = []; const tagIds = []; let res; @@ -102,6 +103,7 @@ const customTagProcessor = async (message, category, destination, contactId) => destType: 'active_campaign', feature: 'transformation', tagEndPoint, + metadata, }); if (res.success === false) { errorHandler(res, 'Failed to fetch already created tags'); @@ -133,6 +135,7 @@ const customTagProcessor = async (message, category, destination, contactId) => endpointPath: `/api/3/tags`, requestMethod: 'GET', module: 'router', + metadata, }); promises.push(resp); } @@ -174,6 +177,7 @@ const customTagProcessor = async (message, category, destination, contactId) => res = await httpPOST(endpoint, requestData, requestOptions, { destType: 'active_campaign', feature: 'transformation', + metadata, }); if (res.success === false) { errorHandler(res, 'Failed to create new tag'); @@ -202,6 +206,7 @@ const customTagProcessor = async (message, category, destination, contactId) => res = httpPOST(endpoint, requestData, requestOptions, { destType: 'active_campaign', feature: 'transformation', + metadata, }); return res; }), @@ -212,7 +217,7 @@ const customTagProcessor = async (message, category, destination, contactId) => }); }; -const customFieldProcessor = async (message, category, destination) => { +const customFieldProcessor = async ({ message, destination, metadata }, category) => { const responseStaging = []; const { fieldEndPoint } = category; // Step - 1 @@ -235,6 +240,7 @@ const customFieldProcessor = async (message, category, destination) => { }; const res = await httpGET(endpoint, requestOptions, { destType: 'active_campaign', + metadata, feature: 'transformation', fieldEndPoint, }); @@ -258,6 +264,7 @@ const customFieldProcessor = async (message, category, destination) => { feature: 'transformation', endpointPath: `/api/3/fields`, requestMethod: 'GET', + metadata, module: 'router', }); promises.push(resp); @@ -317,7 +324,7 @@ const customFieldProcessor = async (message, category, destination) => { return fieldsArrValues; }; -const customListProcessor = async (message, category, destination, contactId) => { +const customListProcessor = async ({ message, destination, metadata }, category, contactId) => { const { mergeListWithContactUrl } = category; // Here we extract the list info from the message const listInfo = get(message?.context?.traits, 'lists') @@ -359,6 +366,7 @@ const customListProcessor = async (message, category, destination, contactId) => endpointPath: mergeListWithContactUrl, requestMethod: 'POST', module: 'router', + metadata, }); promises.push(res); } @@ -374,18 +382,18 @@ const customListProcessor = async (message, category, destination, contactId) => // This the handler func for identify type of events here before we transform the event // and return to rudder server we process the message by calling specific destination apis // for handling tag information and custom field information. -const identifyRequestHandler = async (message, category, destination) => { +const identifyRequestHandler = async ({ message, destination, metadata }, category) => { // create skeleton contact payload let contactPayload = constructPayload(message, MAPPING_CONFIG[category.name]); contactPayload = removeUndefinedAndNullValues(contactPayload); // sync to Active Campaign - const contactId = await syncContact(contactPayload, category, destination); + const contactId = await syncContact(contactPayload, category, { destination, metadata }); // create, and merge tags - await customTagProcessor(message, category, destination, contactId); + await customTagProcessor({ message, destination, metadata }, category, contactId); // add the contact to lists if applicabale - await customListProcessor(message, category, destination, contactId); + await customListProcessor({ message, destination, metadata }, category, contactId); // extract fieldValues to merge with contact - const fieldValues = await customFieldProcessor(message, category, destination); + const fieldValues = await customFieldProcessor({ message, destination, metadata }, category); contactPayload.fieldValues = fieldValues; contactPayload = removeUndefinedAndNullValues(contactPayload); const payload = { @@ -404,7 +412,7 @@ const pageRequestHandler = (message, category, destination) => { return responseBuilderSimple(payload, category, destination); }; -const screenRequestHandler = async (message, category, destination) => { +const screenRequestHandler = async ({ message, destination, metadata }, category) => { // Need to check if the event with same name already exists if not need to create // Retrieve All events from destination // https://developers.activecampaign.com/reference/list-all-event-types @@ -417,6 +425,7 @@ const screenRequestHandler = async (message, category, destination) => { destType: 'active_campaign', feature: 'transformation', endpointPath: `/api/3/eventTrackingEvents`, + metadata, requestMethod: 'GET', module: 'router', }); @@ -445,6 +454,7 @@ const screenRequestHandler = async (message, category, destination) => { }; res = await httpPOST(endpoint, requestData, requestOpt, { destType: 'active_campaign', + metadata, feature: 'transformation', }); if (res.success === false) { @@ -469,7 +479,7 @@ const screenRequestHandler = async (message, category, destination) => { return responseBuilderSimple(payload, category, destination); }; -const trackRequestHandler = async (message, category, destination) => { +const trackRequestHandler = async ({ message, destination, metadata }, category) => { // Need to check if the event with same name already exists if not need to create // Retrieve All events from destination // https://developers.activecampaign.com/reference/list-all-event-types @@ -480,6 +490,7 @@ const trackRequestHandler = async (message, category, destination) => { }, }; let res = await httpGET(endpoint, requestOptions, { + metadata, destType: 'active_campaign', feature: 'transformation', endpointPath: `/api/3/eventTrackingEvents`, @@ -511,6 +522,7 @@ const trackRequestHandler = async (message, category, destination) => { headers: getHeader(destination), }; res = await httpPOST(endpoint, requestData, requestOpt, { + metadata, destType: 'active_campaign', feature: 'transformation', }); @@ -537,7 +549,8 @@ const trackRequestHandler = async (message, category, destination) => { // The main entry point where the message is processed based on what type of event // each scenario is resolved by using specific handler function which does // subsquent processing and transformations and the response is sent to rudder-server -const processEvent = async (message, destination) => { +const processEvent = async (event) => { + const { message, destination } = event; if (!message.type) { throw new InstrumentationError('Message Type is not present. Aborting message.'); } @@ -547,7 +560,7 @@ const processEvent = async (message, destination) => { switch (messageType) { case EventType.IDENTIFY: category = CONFIG_CATEGORIES.IDENTIFY; - response = await identifyRequestHandler(message, category, destination); + response = await identifyRequestHandler(event, category); break; case EventType.PAGE: category = CONFIG_CATEGORIES.PAGE; @@ -555,11 +568,11 @@ const processEvent = async (message, destination) => { break; case EventType.SCREEN: category = CONFIG_CATEGORIES.SCREEN; - response = await screenRequestHandler(message, category, destination); + response = await screenRequestHandler(event, category); break; case EventType.TRACK: category = CONFIG_CATEGORIES.TRACK; - response = await trackRequestHandler(message, category, destination); + response = await trackRequestHandler(event, category); break; default: throw new InstrumentationError('Message type not supported'); @@ -568,7 +581,7 @@ const processEvent = async (message, destination) => { }; const process = async (event) => { - const result = await processEvent(event.message, event.destination); + const result = await processEvent(event); return result; }; diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js index 460f1db565..f2726c3283 100644 --- a/src/v0/destinations/braze/braze.util.test.js +++ b/src/v0/destinations/braze/braze.util.test.js @@ -261,7 +261,7 @@ describe('dedup utility tests', () => { }; // Call the function - const users = await BrazeDedupUtility.doApiLookup(identfierChunks, destination); + const users = await BrazeDedupUtility.doApiLookup(identfierChunks, { destination }); // Check the result expect(users).toEqual([ @@ -399,7 +399,9 @@ describe('dedup utility tests', () => { }, })); - const chunkedUserData = await BrazeDedupUtility.doApiLookup(identifierChunks, destination); + const chunkedUserData = await BrazeDedupUtility.doApiLookup(identifierChunks, { + destination, + }); const result = _.flatMap(chunkedUserData); expect(result).toHaveLength(110); expect(handleHttpRequest).toHaveBeenCalledTimes(3); @@ -455,7 +457,7 @@ describe('dedup utility tests', () => { }, })); - const users = await BrazeDedupUtility.doApiLookup(chunks, destination); + const users = await BrazeDedupUtility.doApiLookup(chunks, { destination }); expect(handleHttpRequest).toHaveBeenCalledTimes(2); // Assert that the first chunk was successful and the second failed @@ -522,7 +524,7 @@ describe('dedup utility tests', () => { [{ alias_name: 'alias1', alias_label: 'rudder_id' }], [{ alias_name: 'alias2', alias_label: 'rudder_id' }], ], - { Config: { restApiKey: 'xyz' } }, + { destination: { Config: { restApiKey: 'xyz' } } }, ); // restore the original implementation of the mocked functions diff --git a/src/v0/destinations/braze/transform.js b/src/v0/destinations/braze/transform.js index 3d6a99d424..155f32c145 100644 --- a/src/v0/destinations/braze/transform.js +++ b/src/v0/destinations/braze/transform.js @@ -203,7 +203,7 @@ function getUserAttributesObject(message, mappingJson, destination) { * @param {*} message * @param {*} destination */ -async function processIdentify(message, destination) { +async function processIdentify({ message, destination, metadata }) { const identifyPayload = getIdentifyPayload(message); const identifyEndpoint = getIdentifyEndpoint(getEndpointFromConfig(destination)); const { processedResponse: brazeIdentifyResp } = await handleHttpRequest( @@ -223,6 +223,7 @@ async function processIdentify(message, destination) { requestMethod: 'POST', module: 'router', endpointPath: '/users/identify', + metadata, }, ); @@ -517,7 +518,7 @@ async function process(event, processParams = { userStore: new Map() }, reqMetad const brazeExternalID = getDestinationExternalID(message, 'brazeExternalId') || message.userId; if (message.anonymousId && brazeExternalID) { - await processIdentify(message, destination); + await processIdentify(event); } else { collectStatsForAliasMissConfigurations(destination.ID); } diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js index 45063d0ba2..00ef308fe9 100644 --- a/src/v0/destinations/braze/util.js +++ b/src/v0/destinations/braze/util.js @@ -142,7 +142,7 @@ const BrazeDedupUtility = { return identfierChunks; }, - async doApiLookup(identfierChunks, destination) { + async doApiLookup(identfierChunks, { destination, metadata }) { return Promise.all( identfierChunks.map(async (ids) => { const externalIdentifiers = ids.filter((id) => id.external_id); @@ -167,6 +167,7 @@ const BrazeDedupUtility = { requestMethod: 'POST', module: 'router', endpointPath: '/users/export/ids', + metadata, }, ); stats.counter('braze_lookup_failure_count', 1, { @@ -189,10 +190,10 @@ const BrazeDedupUtility = { */ async doLookup(inputs) { const lookupStartTime = new Date(); - const { destination } = inputs[0]; + const { destination, metadata } = inputs[0]; const { externalIdsToQuery, aliasIdsToQuery } = this.prepareInputForDedup(inputs); const identfierChunks = this.prepareChunksForDedup(externalIdsToQuery, aliasIdsToQuery); - const chunkedUserData = await this.doApiLookup(identfierChunks, destination); + const chunkedUserData = await this.doApiLookup(identfierChunks, { destination, metadata }); stats.timing('braze_lookup_time', lookupStartTime, { destination_id: destination.Config.destinationId, }); diff --git a/src/v0/destinations/canny/transform.js b/src/v0/destinations/canny/transform.js index f4364e1fb7..02b4e7633d 100644 --- a/src/v0/destinations/canny/transform.js +++ b/src/v0/destinations/canny/transform.js @@ -55,7 +55,7 @@ const identifyResponseBuilder = (message, { Config }) => { return responseBuilder(responseConfgs); }; -const getTrackResponse = async (apiKey, message, operationType) => { +const getTrackResponse = async (apiKey, message, operationType, metadata) => { let endpoint; let responseBody; let contentType; @@ -70,7 +70,7 @@ const getTrackResponse = async (apiKey, message, operationType) => { } payload.apiKey = apiKey; - const voterID = await retrieveUserId(apiKey, message); + const voterID = await retrieveUserId(apiKey, message, metadata); payload.voterID = voterID; endpoint = ConfigCategory.CREATE_VOTE.endpoint; } else if (operationType === 'createPost') { @@ -82,7 +82,7 @@ const getTrackResponse = async (apiKey, message, operationType) => { validateCreatePostFields(payload); payload.apiKey = apiKey; - payload.authorID = await retrieveUserId(apiKey, message); + payload.authorID = await retrieveUserId(apiKey, message, metadata); endpoint = ConfigCategory.CREATE_POST.endpoint; } @@ -97,7 +97,7 @@ const getTrackResponse = async (apiKey, message, operationType) => { return responseBuilder(responseConfgs); }; -const trackResponseBuilder = async (message, { Config }) => { +const trackResponseBuilder = async (message, { Config }, metadata) => { const { apiKey, eventsToEvents } = Config; const configuredEventsMap = getHashFromArrayWithDuplicate(eventsToEvents); @@ -113,7 +113,7 @@ const trackResponseBuilder = async (message, { Config }) => { // eslint-disable-next-line no-restricted-syntax for (const destinationEvent of destinationEvents) { // eslint-disable-next-line no-await-in-loop - const response = await getTrackResponse(apiKey, message, destinationEvent); + const response = await getTrackResponse(apiKey, message, destinationEvent, metadata); responseArray.push(response); } } @@ -122,7 +122,7 @@ const trackResponseBuilder = async (message, { Config }) => { return responseArray; }; -const processEvent = (message, destination) => { +const processEvent = async (message, destination, metadata) => { if (!destination.Config.apiKey) { throw new ConfigurationError('API Key is not present. Aborting message.'); } @@ -137,7 +137,7 @@ const processEvent = (message, destination) => { response = identifyResponseBuilder(message, destination); break; case EventType.TRACK: - response = trackResponseBuilder(message, destination); + response = await trackResponseBuilder(message, destination, metadata); break; default: throw new InstrumentationError('Message type not supported'); @@ -145,7 +145,7 @@ const processEvent = (message, destination) => { return response; }; -const process = (event) => processEvent(event.message, event.destination); +const process = async (event) => processEvent(event.message, event.destination, event.metadata); const processRouterDest = async (inputs, reqMetadata) => { const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); diff --git a/src/v0/destinations/canny/util.js b/src/v0/destinations/canny/util.js index 1d03eed4b9..eeefc141db 100644 --- a/src/v0/destinations/canny/util.js +++ b/src/v0/destinations/canny/util.js @@ -13,7 +13,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); * @param message * @returns canny userId */ -const retrieveUserId = async (apiKey, message) => { +const retrieveUserId = async (apiKey, message, metadata) => { const cannyId = getDestinationExternalID(message, 'cannyUserId'); if (cannyId) { return cannyId; @@ -43,6 +43,7 @@ const retrieveUserId = async (apiKey, message) => { qs.stringify(requestBody), { headers }, { + metadata, destType: 'canny', feature: 'transformation', endpointPath: `/v1/users/retrieve`, diff --git a/src/v0/destinations/clickup/transform.js b/src/v0/destinations/clickup/transform.js index 0637d65bd4..799fdf7e7b 100644 --- a/src/v0/destinations/clickup/transform.js +++ b/src/v0/destinations/clickup/transform.js @@ -33,7 +33,7 @@ const responseBuilder = async (payload, listId, apiToken) => { throw new TransformationError('Something went wrong while constructing the payload'); }; -const trackResponseBuilder = async (message, destination) => { +const trackResponseBuilder = async (message, destination, metadata) => { const { apiToken, keyToCustomFieldName } = destination.Config; const { properties } = message; const externalListId = getDestinationExternalID(message, 'clickUpListId'); @@ -45,6 +45,7 @@ const trackResponseBuilder = async (message, destination) => { properties, listId, apiToken, + metadata, ); let payload = constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK.name]); @@ -55,7 +56,7 @@ const trackResponseBuilder = async (message, destination) => { return responseBuilder(payload, listId, apiToken); }; -const processEvent = async (message, destination) => { +const processEvent = async (message, destination, metadata) => { if (!message.type) { throw new InstrumentationError('Event type is required'); } @@ -64,13 +65,13 @@ const processEvent = async (message, destination) => { const messageType = message.type.toLowerCase(); if (messageType === EventType.TRACK) { - return trackResponseBuilder(message, destination); + return trackResponseBuilder(message, destination, metadata); } throw new InstrumentationError(`Event type "${messageType}" is not supported`); }; -const process = async (event) => processEvent(event.message, event.destination); +const process = async (event) => processEvent(event.message, event.destination, event.metadata); const processRouterDest = async (inputs, reqMetadata) => { const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); diff --git a/src/v0/destinations/clickup/util.js b/src/v0/destinations/clickup/util.js index 74e961906c..5da4192b5b 100644 --- a/src/v0/destinations/clickup/util.js +++ b/src/v0/destinations/clickup/util.js @@ -206,7 +206,7 @@ const getListOfAssignees = (message, type) => { * @param {*} apiToken * @returns */ -const retrieveCustomFields = async (listId, apiToken) => { +const retrieveCustomFields = async (listId, apiToken, metadata) => { const endpoint = getCustomFieldsEndPoint(listId); const requestOptions = { headers: { @@ -220,6 +220,7 @@ const retrieveCustomFields = async (listId, apiToken) => { endpointPath: '/list/listId/field', requestMethod: 'GET', module: 'router', + metadata, }); const processedCustomFieldsResponse = processAxiosResponse(customFieldsResponse); @@ -278,11 +279,17 @@ const extractUIMappedCustomFieldDetails = ( * @param {*} apiToken * @returns [{"id":"b0f40a94-ea2a-4998-a514-8074d0eddcde","value":"https://www.rudderstack.com/"}] */ -const customFieldsBuilder = async (keyToCustomFieldName, properties, listId, apiToken) => { +const customFieldsBuilder = async ( + keyToCustomFieldName, + properties, + listId, + apiToken, + metadata, +) => { const responseArray = []; if (properties && keyToCustomFieldName) { // retrieve available clickup custom field for the given list - const availableCustomFields = await retrieveCustomFields(listId, apiToken); + const availableCustomFields = await retrieveCustomFields(listId, apiToken, metadata); // convert array to hashMap with key as field name and value as custom field object const availableCustomFieldsMap = getHashFromArrayWithValueAsObject( availableCustomFields, diff --git a/src/v0/destinations/custify/transform.js b/src/v0/destinations/custify/transform.js index 6b08be1c56..d13a476a68 100644 --- a/src/v0/destinations/custify/transform.js +++ b/src/v0/destinations/custify/transform.js @@ -19,7 +19,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); * @param {*} destination * @returns */ -const validateAndBuildResponse = async (message, destination) => { +const validateAndBuildResponse = async ({ message, destination, metadata }) => { const messageType = message.type.toLowerCase(); const response = defaultRequestConfig(); let responseBody; @@ -34,7 +34,7 @@ const validateAndBuildResponse = async (message, destination) => { category = ConfigCategory.TRACK; break; case EventType.GROUP: - responseBody = await processGroup(message, destination); + responseBody = await processGroup(message, destination, metadata); category = ConfigCategory.GROUP_USER; break; default: @@ -57,14 +57,14 @@ const validateAndBuildResponse = async (message, destination) => { return response; }; -const processSingleMessage = async (message, destination) => { +const processSingleMessage = async ({ message, destination, metadata }) => { if (!message.type) { throw new InstrumentationError('Message Type is not present. Ignoring message.'); } - return validateAndBuildResponse(message, destination); + return validateAndBuildResponse({ message, destination, metadata }); }; -const process = (event) => processSingleMessage(event.message, event.destination); +const process = (event) => processSingleMessage(event); const processRouterDest = async (inputs, reqMetadata) => { const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); diff --git a/src/v0/destinations/custify/util.js b/src/v0/destinations/custify/util.js index b6f3446503..f35dd4dd23 100644 --- a/src/v0/destinations/custify/util.js +++ b/src/v0/destinations/custify/util.js @@ -27,7 +27,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); * @param {*} Config * @api https://docs.custify.com/#tag/Company/paths/~1company/post */ -const createUpdateCompany = async (companyPayload, Config) => { +const createUpdateCompany = async (companyPayload, Config, metadata) => { const companyResponse = await httpPOST( ConfigCategory.GROUP_COMPANY.endpoint, companyPayload, @@ -43,6 +43,7 @@ const createUpdateCompany = async (companyPayload, Config) => { endpointPath: `/company`, requestMethod: 'POST', module: 'router', + metadata, }, ); const processedCompanyResponse = processAxiosResponse(companyResponse); @@ -187,7 +188,7 @@ const processTrack = (message, { Config }) => { * @api https://docs.custify.com/#tag/People/paths/~1people/post * @api https://docs.custify.com/#tag/Company/paths/~1company/post */ -const processGroup = async (message, { Config }) => { +const processGroup = async (message, { Config }, metadata) => { let companyPayload = constructPayload(message, MappingConfig[ConfigCategory.GROUP_COMPANY.name]); if (!companyPayload.company_id) { throw new InstrumentationError('groupId Id is mandatory'); @@ -205,7 +206,7 @@ const processGroup = async (message, { Config }) => { }); } companyPayload = removeUndefinedAndNullValues(companyPayload); - await createUpdateCompany(companyPayload, Config); + await createUpdateCompany(companyPayload, Config, metadata); const userPayload = constructPayload(message, MappingConfig[ConfigCategory.GROUP_USER.name]); const { sendAnonymousId } = Config; if (sendAnonymousId && !userPayload.user_id) { diff --git a/src/v0/destinations/delighted/transform.js b/src/v0/destinations/delighted/transform.js index cf80dc878d..c560dba03d 100644 --- a/src/v0/destinations/delighted/transform.js +++ b/src/v0/destinations/delighted/transform.js @@ -77,7 +77,7 @@ const identifyResponseBuilder = (message, { Config }) => { return response; }; -const trackResponseBuilder = async (message, { Config }) => { +const trackResponseBuilder = async (message, { Config }, metadata) => { // checks if the event is valid if not throws error else nothing const isValidEvent = eventValidity(Config, message); if (!isValidEvent) { @@ -94,7 +94,7 @@ const trackResponseBuilder = async (message, { Config }) => { const { userIdType, userIdValue } = isValidUserIdOrError(channel, userId); // checking if user already exists or not, throw error if it doesn't - const check = await userValidity(channel, Config, userId); + const check = await userValidity({ channel, Config, userId, metadata }); if (!check) { throw new NetworkInstrumentationError(`user ${userId} doesn't exist`); @@ -167,7 +167,7 @@ const aliasResponseBuilder = (message, { Config }) => { }; const process = async (event) => { - const { message, destination } = event; + const { message, destination, metadata } = event; if (!message.type) { throw new InstrumentationError('Message Type is not present. Aborting message.'); } @@ -184,7 +184,7 @@ const process = async (event) => { response = identifyResponseBuilder(message, destination); break; case EventType.TRACK: - response = await trackResponseBuilder(message, destination); + response = await trackResponseBuilder(message, destination, metadata); break; case EventType.ALIAS: response = aliasResponseBuilder(message, destination); diff --git a/src/v0/destinations/delighted/util.js b/src/v0/destinations/delighted/util.js index 53f416b48d..99a0923dc9 100644 --- a/src/v0/destinations/delighted/util.js +++ b/src/v0/destinations/delighted/util.js @@ -61,7 +61,7 @@ const getErrorStatus = (status) => { return errStatus; }; -const userValidity = async (channel, Config, userId) => { +const userValidity = async ({ channel, Config, userId, metadata }) => { const paramsdata = {}; if (channel === 'email') { paramsdata.email = userId; @@ -81,6 +81,7 @@ const userValidity = async (channel, Config, userId) => { params: paramsdata, }, { + metadata, destType: 'delighted', feature: 'transformation', requestMethod: 'GET', diff --git a/src/v0/destinations/freshmarketer/transform.js b/src/v0/destinations/freshmarketer/transform.js index aa0e03811d..9cf9757441 100644 --- a/src/v0/destinations/freshmarketer/transform.js +++ b/src/v0/destinations/freshmarketer/transform.js @@ -70,7 +70,7 @@ const identifyResponseBuilder = (message, { Config }) => { * @param {*} Config * @returns */ -const trackResponseBuilder = async (message, { Config }, event) => { +const trackResponseBuilder = async ({ message, destination: { Config }, metadata }, event) => { if (!event) { throw new InstrumentationError('Event name is required for track call.'); } @@ -85,11 +85,12 @@ const trackResponseBuilder = async (message, { Config }, event) => { payload, message, Config, + metadata, ); break; } case 'lifecycle_stage': { - response.body.JSON = await UpdateContactWithLifeCycleStage(message, Config); + response.body.JSON = await UpdateContactWithLifeCycleStage(message, Config, metadata); response.endpoint = `https://${Config.domain}${CONFIG_CATEGORIES.IDENTIFY.baseUrl}`; break; } @@ -111,7 +112,7 @@ const trackResponseBuilder = async (message, { Config }, event) => { * @param {*} Config * @returns */ -const groupResponseBuilder = async (message, { Config }) => { +const groupResponseBuilder = async ({ message, destination: { Config }, metadata }) => { const groupType = get(message, 'traits.groupType'); if (!groupType) { throw new InstrumentationError('groupType is required for Group call'); @@ -130,7 +131,7 @@ const groupResponseBuilder = async (message, { Config }) => { response = updateAccountWOContact(payload, Config); break; } - const accountDetails = await getUserAccountDetails(payload, userEmail, Config); + const accountDetails = await getUserAccountDetails(payload, userEmail, Config, metadata); response = identifyResponseConfig(Config); response.body.JSON.contact = { sales_accounts: accountDetails }; response.body.JSON.unique_identifier = { emails: userEmail }; @@ -143,7 +144,7 @@ const groupResponseBuilder = async (message, { Config }) => { 'email is required for adding in the marketing lists. Aborting!', ); } - const userDetails = await getContactsDetails(userEmail, Config); + const userDetails = await getContactsDetails(userEmail, Config, metadata); const userId = userDetails.response?.contact?.id; if (!userId) { throw new NetworkInstrumentationError('Failed in fetching userId. Aborting!'); @@ -153,7 +154,7 @@ const groupResponseBuilder = async (message, { Config }) => { if (listId) { response = updateContactWithList(userId, listId, Config); } else if (listName) { - listId = await createOrUpdateListDetails(listName, Config); + listId = await createOrUpdateListDetails(listName, Config, metadata); if (!listId) { throw new NetworkInstrumentationError('Failed in fetching listId. Aborting!'); } @@ -198,7 +199,7 @@ function eventMappingHandler(message, destination) { return [...mappedEvents]; } -const processEvent = async (message, destination) => { +const processEvent = async ({ message, destination, metadata }) => { if (!message.type) { throw new InstrumentationError('Message Type is not present. Aborting message.'); } @@ -213,16 +214,19 @@ const processEvent = async (message, destination) => { if (mappedEvents.length > 0) { response = await Promise.all( mappedEvents.map(async (mappedEvent) => - trackResponseBuilder(message, destination, mappedEvent), + trackResponseBuilder({ message, destination, metadata }, mappedEvent), ), ); } else { - response = await trackResponseBuilder(message, destination, get(message, 'event')); + response = await trackResponseBuilder( + { message, destination, metadata }, + get(message, 'event'), + ); } break; } case EventType.GROUP: - response = await groupResponseBuilder(message, destination); + response = await groupResponseBuilder({ message, destination, metadata }); break; default: throw new InstrumentationError(`message type ${messageType} not supported`); @@ -230,7 +234,7 @@ const processEvent = async (message, destination) => { return response; }; -const process = async (event) => processEvent(event.message, event.destination); +const process = async (event) => processEvent(event); const processRouterDest = async (inputs, reqMetadata) => { const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); diff --git a/src/v0/destinations/freshmarketer/utils.js b/src/v0/destinations/freshmarketer/utils.js index c80711ff8d..879cd6eace 100644 --- a/src/v0/destinations/freshmarketer/utils.js +++ b/src/v0/destinations/freshmarketer/utils.js @@ -37,7 +37,7 @@ const getHeaders = (apiKey) => { * @returns * ref: https://developers.freshworks.com/crm/api/#upsert_an_account */ -const createUpdateAccount = async (payload, Config) => { +const createUpdateAccount = async (payload, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -52,6 +52,7 @@ const createUpdateAccount = async (payload, Config) => { endpointPath: `/crm/sales/api/sales_accounts/upsert`, requestMethod: 'POST', module: 'router', + metadata, }); accountResponse = processAxiosResponse(accountResponse); if (accountResponse.status !== 200 && accountResponse.status !== 201) { @@ -80,7 +81,7 @@ const createUpdateAccount = async (payload, Config) => { * @returns * ref: https://developers.freshworks.com/crm/api/#upsert_a_contact */ -const getUserAccountDetails = async (payload, userEmail, Config) => { +const getUserAccountDetails = async (payload, userEmail, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -99,6 +100,7 @@ const getUserAccountDetails = async (payload, userEmail, Config) => { endpointPath: `crm/sales/api/contacts/upsert?include=sales_accounts`, requestMethod: 'POST', module: 'router', + metadata, }); userSalesAccountResponse = processAxiosResponse(userSalesAccountResponse); if (userSalesAccountResponse.status !== 200 && userSalesAccountResponse.status !== 201) { @@ -117,7 +119,7 @@ const getUserAccountDetails = async (payload, userEmail, Config) => { if (!accountDetails) { throw new NetworkInstrumentationError('Fails in fetching user accountDetails'); } - const accountId = await createUpdateAccount(payload, Config); + const accountId = await createUpdateAccount(payload, Config, metadata); const accountDetail = { id: accountId, is_primary: false, @@ -139,7 +141,7 @@ const getUserAccountDetails = async (payload, userEmail, Config) => { * @returns * ref: https://developers.freshworks.com/crm/api/#upsert_an_account */ -const createOrUpdateListDetails = async (listName, Config) => { +const createOrUpdateListDetails = async (listName, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -151,6 +153,7 @@ const createOrUpdateListDetails = async (listName, Config) => { endpointPath: `/crm/sales/api/lists`, requestMethod: 'GET', module: 'router', + metadata, }); listResponse = processAxiosResponse(listResponse); if (listResponse.status !== 200) { @@ -173,6 +176,7 @@ const createOrUpdateListDetails = async (listName, Config) => { endpointPath: `/crm/sales/api/lists`, requestMethod: 'POST', module: 'router', + metadata, }); listResponse = processAxiosResponse(listResponse); if (listResponse.status !== 200) { @@ -231,7 +235,7 @@ const updateContactWithList = (userId, listId, Config) => { * @returns * ref: https://developers.freshworks.com/crm/api/#upsert_a_contact */ -const getContactsDetails = async (userEmail, Config) => { +const getContactsDetails = async (userEmail, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -250,6 +254,7 @@ const getContactsDetails = async (userEmail, Config) => { endpointPath: `/crm/sales/api/contacts/upsert`, requestMethod: 'POST', module: 'router', + metadata, }); userResponse = processAxiosResponse(userResponse); if (userResponse.status !== 200 && userResponse.status !== 201) { @@ -276,8 +281,14 @@ const getContactsDetails = async (userEmail, Config) => { * returns */ -const responseBuilderWithContactDetails = async (email, Config, payload, salesActivityTypeId) => { - const userDetails = await getContactsDetails(email, Config); +const responseBuilderWithContactDetails = async ( + email, + Config, + payload, + salesActivityTypeId, + metadata, +) => { + const userDetails = await getContactsDetails(email, Config, metadata); const userId = userDetails.response?.contact?.id; if (!userId) { throw new NetworkInstrumentationError('Failed in fetching userId. Aborting!'); @@ -295,7 +306,7 @@ const responseBuilderWithContactDetails = async (email, Config, payload, salesAc * @param {*} Config - headers, apiKey... * ref: https://developers.freshworks.com/crm/api/#admin_configuration */ -const UpdateContactWithLifeCycleStage = async (message, Config) => { +const UpdateContactWithLifeCycleStage = async (message, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -326,6 +337,7 @@ const UpdateContactWithLifeCycleStage = async (message, Config) => { endpointPath: `/crm/sales/api/selector/lifecycle_stages`, requestMethod: 'GET', module: 'router', + metadata, }); lifeCycleStagesResponse = processAxiosResponse(lifeCycleStagesResponse); if (lifeCycleStagesResponse.status !== 200) { @@ -368,7 +380,7 @@ const UpdateContactWithLifeCycleStage = async (message, Config) => { * @param {*} Config - headers, apiKey... * ref: https://developers.freshworks.com/crm/api/#list_all_sales_activities */ -const UpdateContactWithSalesActivity = async (payload, message, Config) => { +const UpdateContactWithSalesActivity = async (payload, message, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -414,6 +426,7 @@ const UpdateContactWithSalesActivity = async (payload, message, Config) => { endpointPath: `/crm/sales/api/selector/sales_activity_types`, requestMethod: 'GET', module: 'router', + metadata, }); salesActivityResponse = processAxiosResponse(salesActivityResponse); if (salesActivityResponse.status !== 200) { @@ -452,6 +465,7 @@ const UpdateContactWithSalesActivity = async (payload, message, Config) => { Config, payload, salesActivityDetails.id, + metadata, ); } diff --git a/src/v0/destinations/freshsales/transform.js b/src/v0/destinations/freshsales/transform.js index 096a2d749c..37c081be49 100644 --- a/src/v0/destinations/freshsales/transform.js +++ b/src/v0/destinations/freshsales/transform.js @@ -66,7 +66,7 @@ const identifyResponseBuilder = (message, { Config }) => { * @param {*} Config * @returns */ -const trackResponseBuilder = async (message, { Config }, event) => { +const trackResponseBuilder = async ({ message, destination: { Config }, metadata }, event) => { let payload; const response = defaultRequestConfig(); @@ -78,11 +78,12 @@ const trackResponseBuilder = async (message, { Config }, event) => { payload, message, Config, + metadata, ); break; } case 'lifecycle_stage': { - response.body.JSON = await UpdateContactWithLifeCycleStage(message, Config); + response.body.JSON = await UpdateContactWithLifeCycleStage(message, Config, metadata); response.endpoint = `https://${Config.domain}${CONFIG_CATEGORIES.IDENTIFY.baseUrl}`; break; } @@ -100,7 +101,7 @@ const trackResponseBuilder = async (message, { Config }, event) => { * @param {*} Config * @returns */ -const groupResponseBuilder = async (message, { Config }) => { +const groupResponseBuilder = async ({ message, destination: { Config }, metadata }) => { const payload = constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.GROUP.name]); if (!payload) { // fail-safety for developer error @@ -114,7 +115,7 @@ const groupResponseBuilder = async (message, { Config }) => { return updateAccountWOContact(payload, Config); } - const accountDetails = await getUserAccountDetails(payload, userEmail, Config); + const accountDetails = await getUserAccountDetails(payload, userEmail, Config, metadata); const responseIdentify = identifyResponseConfig(Config); responseIdentify.body.JSON.contact = { sales_accounts: accountDetails }; responseIdentify.body.JSON.unique_identifier = { emails: userEmail }; @@ -146,7 +147,8 @@ function eventMappingHandler(message, destination) { return [...mappedEvents]; } -const processEvent = async (message, destination) => { +const processEvent = async (event) => { + const { message, destination, metadata } = event; if (!message.type) { throw new InstrumentationError('Message Type is not present. Aborting message.'); } @@ -162,25 +164,28 @@ const processEvent = async (message, destination) => { if (mappedEvents.length > 0) { const respList = await Promise.all( mappedEvents.map(async (mappedEvent) => - trackResponseBuilder(message, destination, mappedEvent), + trackResponseBuilder({ message, destination, metadata }, mappedEvent), ), ); response = respList; } else { - response = await trackResponseBuilder(message, destination, get(message, 'event')); + response = await trackResponseBuilder( + { message, destination, metadata }, + get(message, 'event'), + ); } break; } case EventType.GROUP: - response = await groupResponseBuilder(message, destination); + response = await groupResponseBuilder({ message, destination, metadata }); break; default: throw new InstrumentationError(`message type ${messageType} not supported`); } return response; }; -const process = async (event) => processEvent(event.message, event.destination); +const process = async (event) => processEvent(event); const processRouterDest = async (inputs, reqMetadata) => { const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); diff --git a/src/v0/destinations/freshsales/utils.js b/src/v0/destinations/freshsales/utils.js index 977bde0abb..14cca0a3d6 100644 --- a/src/v0/destinations/freshsales/utils.js +++ b/src/v0/destinations/freshsales/utils.js @@ -35,7 +35,7 @@ const getHeaders = (apiKey) => { * @returns * ref: https://developers.freshworks.com/crm/api/#upsert_an_account */ -const createUpdateAccount = async (payload, Config) => { +const createUpdateAccount = async (payload, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -50,6 +50,7 @@ const createUpdateAccount = async (payload, Config) => { endpointPath: `/crm/sales/api/sales_accounts/upsert`, requestMethod: 'POST', module: 'router', + metadata, }); accountResponse = processAxiosResponse(accountResponse); if (accountResponse.status !== 200 && accountResponse.status !== 201) { @@ -77,7 +78,7 @@ const createUpdateAccount = async (payload, Config) => { * @returns * ref: https://developers.freshworks.com/crm/api/#upsert_a_contact */ -const getUserAccountDetails = async (payload, userEmail, Config) => { +const getUserAccountDetails = async (payload, userEmail, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -96,6 +97,7 @@ const getUserAccountDetails = async (payload, userEmail, Config) => { endpointPath: `/crm/sales/api/contacts/upsert?include=sales_accounts`, requestMethod: 'POST', module: 'router', + metadata, }); userSalesAccountResponse = processAxiosResponse(userSalesAccountResponse); if (userSalesAccountResponse.status !== 200 && userSalesAccountResponse.status !== 201) { @@ -114,7 +116,7 @@ const getUserAccountDetails = async (payload, userEmail, Config) => { if (!accountDetails) { throw new NetworkInstrumentationError('Fails in fetching user accountDetails'); } - const accountId = await createUpdateAccount(payload, Config); + const accountId = await createUpdateAccount(payload, Config, metadata); const accountDetail = { id: accountId, is_primary: false, @@ -135,7 +137,7 @@ const getUserAccountDetails = async (payload, userEmail, Config) => { * @returns * ref: https://developers.freshworks.com/crm/api/#upsert_a_contact */ -const getContactsDetails = async (userEmail, Config) => { +const getContactsDetails = async (userEmail, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -154,6 +156,7 @@ const getContactsDetails = async (userEmail, Config) => { endpointPath: `/crm/sales/api/contacts/upsert`, requestMethod: 'POST', module: 'router', + metadata, }); userResponse = processAxiosResponse(userResponse); if (userResponse.status !== 200 && userResponse.status !== 201) { @@ -180,8 +183,14 @@ const getContactsDetails = async (userEmail, Config) => { * returns */ -const responseBuilderWithContactDetails = async (email, Config, payload, salesActivityTypeId) => { - const userDetails = await getContactsDetails(email, Config); +const responseBuilderWithContactDetails = async ( + email, + Config, + payload, + salesActivityTypeId, + metadata, +) => { + const userDetails = await getContactsDetails(email, Config, metadata); const userId = userDetails.response?.contact?.id; if (!userId) { throw new NetworkInstrumentationError('Failed in fetching userId. Aborting!', userDetails); @@ -201,7 +210,7 @@ const responseBuilderWithContactDetails = async (email, Config, payload, salesAc * @param {*} Config - headers, apiKey... * ref: https://developers.freshworks.com/crm/api/#list_all_sales_activities */ -const UpdateContactWithSalesActivity = async (payload, message, Config) => { +const UpdateContactWithSalesActivity = async (payload, message, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -230,11 +239,12 @@ const UpdateContactWithSalesActivity = async (payload, message, Config) => { sales_activity_type_id: payload.sales_activity_type_id, }; } else { - responseBody = responseBuilderWithContactDetails( + responseBody = await responseBuilderWithContactDetails( email, Config, payload, payload.sales_activity_type_id, + metadata, ); } return responseBody; @@ -247,6 +257,7 @@ const UpdateContactWithSalesActivity = async (payload, message, Config) => { endpointPath: `/crm/sales/api/sales_activity_types`, requestMethod: 'GET', module: 'router', + metadata, }); salesActivityResponse = processAxiosResponse(salesActivityResponse); if (salesActivityResponse.status !== 200) { @@ -285,6 +296,7 @@ const UpdateContactWithSalesActivity = async (payload, message, Config) => { Config, payload, salesActivityDetails.id, + metadata, ); } @@ -298,7 +310,7 @@ const UpdateContactWithSalesActivity = async (payload, message, Config) => { * @param {*} Config - headers, apiKey... * ref: https://developers.freshworks.com/crm/api/#admin_configuration */ -const UpdateContactWithLifeCycleStage = async (message, Config) => { +const UpdateContactWithLifeCycleStage = async (message, Config, metadata) => { const requestOptions = { headers: getHeaders(Config.apiKey), }; @@ -329,6 +341,7 @@ const UpdateContactWithLifeCycleStage = async (message, Config) => { endpointPath: `/crm/sales/api/lifecycle_stages`, requestMethod: 'GET', module: 'router', + metadata, }); lifeCycleStagesResponse = processAxiosResponse(lifeCycleStagesResponse); if (lifeCycleStagesResponse.status !== 200) { diff --git a/src/v0/destinations/hs/HSTransform-v1.js b/src/v0/destinations/hs/HSTransform-v1.js index 51feebea74..ed94bd7c17 100644 --- a/src/v0/destinations/hs/HSTransform-v1.js +++ b/src/v0/destinations/hs/HSTransform-v1.js @@ -51,7 +51,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); * @param {*} propertyMap * @returns */ -const processLegacyIdentify = async (message, destination, propertyMap) => { +const processLegacyIdentify = async ({ message, destination, metadata }, propertyMap) => { const { Config } = destination; let traits = getFieldValueFromMessage(message, 'traits'); const mappedToDestination = get(message, MappedToDestinationKey); @@ -82,7 +82,7 @@ const processLegacyIdentify = async (message, destination, propertyMap) => { response.method = defaultPatchRequestConfig.requestMethod; } - traits = await populateTraits(propertyMap, traits, destination); + traits = await populateTraits(propertyMap, traits, destination, metadata); response.body.JSON = removeUndefinedAndNullValues({ properties: traits }); response.source = 'rETL'; response.operation = operation; @@ -92,7 +92,10 @@ const processLegacyIdentify = async (message, destination, propertyMap) => { } const { email } = traits; - const userProperties = await getTransformedJSON(message, destination, propertyMap); + const userProperties = await getTransformedJSON( + { message, destination, metadata }, + propertyMap, + ); const payload = { properties: formatPropertyValueForIdentify(userProperties), @@ -134,7 +137,7 @@ const processLegacyIdentify = async (message, destination, propertyMap) => { * @param {*} propertyMap * @returns */ -const processLegacyTrack = async (message, destination, propertyMap) => { +const processLegacyTrack = async ({ message, destination, metadata }, propertyMap) => { const { Config } = destination; if (!Config.hubID) { @@ -151,7 +154,7 @@ const processLegacyTrack = async (message, destination, propertyMap) => { id: getDestinationExternalID(message, 'hubspotId'), }; - const userProperties = await getTransformedJSON(message, destination, propertyMap); + const userProperties = await getTransformedJSON({ message, destination, metadata }, propertyMap); const payload = { ...parameters, ...userProperties }; const params = removeUndefinedAndNullValues(payload); diff --git a/src/v0/destinations/hs/HSTransform-v2.js b/src/v0/destinations/hs/HSTransform-v2.js index 71f080205a..3dd9f87ea4 100644 --- a/src/v0/destinations/hs/HSTransform-v2.js +++ b/src/v0/destinations/hs/HSTransform-v2.js @@ -69,7 +69,7 @@ const addHsAuthentication = (response, Config) => { * @param {*} propertyMap * @returns */ -const processIdentify = async (message, destination, propertyMap) => { +const processIdentify = async ({ message, destination, metadata }, propertyMap) => { const { Config } = destination; let traits = getFieldValueFromMessage(message, 'traits'); const mappedToDestination = get(message, MappedToDestinationKey); @@ -126,7 +126,7 @@ const processIdentify = async (message, destination, propertyMap) => { response.method = defaultPatchRequestConfig.requestMethod; } - traits = await populateTraits(propertyMap, traits, destination); + traits = await populateTraits(propertyMap, traits, destination, metadata); response.body.JSON = removeUndefinedAndNullValues({ properties: traits }); response.source = 'rETL'; response.operation = operation; @@ -139,10 +139,10 @@ const processIdentify = async (message, destination, propertyMap) => { // if contactId is not provided then search if (!contactId) { - contactId = await searchContacts(message, destination); + contactId = await searchContacts(message, destination, metadata); } - const properties = await getTransformedJSON(message, destination, propertyMap); + const properties = await getTransformedJSON({ message, destination, metadata }, propertyMap); const payload = { properties, @@ -188,7 +188,7 @@ const processIdentify = async (message, destination, propertyMap) => { * @param {*} destination * @returns */ -const processTrack = async (message, destination) => { +const processTrack = async ({ message, destination }) => { const { Config } = destination; let payload = constructPayload(message, mappingConfig[ConfigCategory.TRACK.name]); diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js index fe05d45fbb..6cf69e3c3b 100644 --- a/src/v0/destinations/hs/transform.js +++ b/src/v0/destinations/hs/transform.js @@ -21,7 +21,7 @@ const { validateDestinationConfig, } = require('./util'); -const processSingleMessage = async (message, destination, propertyMap) => { +const processSingleMessage = async ({ message, destination, metadata }, propertyMap) => { if (!message.type) { throw new InstrumentationError('Message type is not present. Aborting message.'); } @@ -34,18 +34,18 @@ const processSingleMessage = async (message, destination, propertyMap) => { case EventType.IDENTIFY: { response = []; if (destination.Config.apiVersion === API_VERSION.v3) { - response.push(await processIdentify(message, destination, propertyMap)); + response.push(await processIdentify({ message, destination, metadata }, propertyMap)); } else { // Legacy API - response.push(await processLegacyIdentify(message, destination, propertyMap)); + response.push(await processLegacyIdentify({ message, destination, metadata }, propertyMap)); } break; } case EventType.TRACK: if (destination.Config.apiVersion === API_VERSION.v3) { - response = await processTrack(message, destination, propertyMap); + response = await processTrack({ message, destination }, propertyMap); } else { - response = await processLegacyTrack(message, destination, propertyMap); + response = await processLegacyTrack({ message, destination, metadata }, propertyMap); } break; default: @@ -57,20 +57,24 @@ const processSingleMessage = async (message, destination, propertyMap) => { // has been deprecated - using routerTransform for both the versions const process = async (event) => { - const { destination, message } = event; + const { destination, message, metadata } = event; const mappedToDestination = get(message, MappedToDestinationKey); let events = []; events = [event]; if (mappedToDestination && GENERIC_TRUE_VALUES.includes(mappedToDestination?.toString())) { // get info about existing objects and splitting accordingly. - events = await splitEventsForCreateUpdate([event], destination); + events = await splitEventsForCreateUpdate([event], destination, metadata); } - return processSingleMessage(events[0].message, events[0].destination); + return processSingleMessage({ + message: events[0].message, + destination: events[0].destination, + metadata: events[0].metadata || metadata, + }); }; const processBatchRouter = async (inputs, reqMetadata) => { let tempInputs = inputs; // using the first destination config for transforming the batch - const { destination } = tempInputs[0]; + const { destination, metadata } = tempInputs[0]; let propertyMap; const mappedToDestination = get(tempInputs[0].message, MappedToDestinationKey); const { objectType } = getDestinationExternalIDInfoForRetl(tempInputs[0].message, 'HS'); @@ -82,9 +86,9 @@ const processBatchRouter = async (inputs, reqMetadata) => { if (mappedToDestination && GENERIC_TRUE_VALUES.includes(mappedToDestination?.toString())) { // skip splitting the batches to inserts and updates if object it is an association if (objectType.toLowerCase() !== 'association') { - propertyMap = await getProperties(destination); + propertyMap = await getProperties(destination, metadata); // get info about existing objects and splitting accordingly. - tempInputs = await splitEventsForCreateUpdate(tempInputs, destination); + tempInputs = await splitEventsForCreateUpdate(tempInputs, destination, metadata); } } else { // reduce the no. of calls for properties endpoint @@ -92,7 +96,7 @@ const processBatchRouter = async (inputs, reqMetadata) => { (input) => fetchFinalSetOfTraits(input.message) !== undefined, ); if (traitsFound) { - propertyMap = await getProperties(destination); + propertyMap = await getProperties(destination, metadata); } } } catch (error) { @@ -118,8 +122,7 @@ const processBatchRouter = async (inputs, reqMetadata) => { } else { // event is not transformed let receivedResponse = await processSingleMessage( - input.message, - destination, + { message: input.message, destination, metadata: input.metadata }, propertyMap, ); diff --git a/src/v0/destinations/hs/util.js b/src/v0/destinations/hs/util.js index b30207fe15..38b2e636b9 100644 --- a/src/v0/destinations/hs/util.js +++ b/src/v0/destinations/hs/util.js @@ -90,7 +90,7 @@ const fetchFinalSetOfTraits = (message) => { * @param {*} destination * @returns */ -const getProperties = async (destination) => { +const getProperties = async (destination, metadata) => { let hubspotPropertyMap = {}; let hubspotPropertyMapResponse; const { Config } = destination; @@ -110,6 +110,7 @@ const getProperties = async (destination) => { endpointPath: `/properties/v1/contacts/properties`, requestMethod: 'GET', module: 'router', + metadata, }); hubspotPropertyMapResponse = processAxiosResponse(hubspotPropertyMapResponse); } else { @@ -124,6 +125,7 @@ const getProperties = async (destination) => { endpointPath: `/properties/v1/contacts/properties?hapikey`, requestMethod: 'GET', module: 'router', + metadata, }, ); hubspotPropertyMapResponse = processAxiosResponse(hubspotPropertyMapResponse); @@ -208,7 +210,7 @@ const getUTCMidnightTimeStampValue = (propValue) => { * @param {*} propertyMap * @returns */ -const getTransformedJSON = async (message, destination, propertyMap) => { +const getTransformedJSON = async ({ message, destination, metadata }, propertyMap) => { let rawPayload = {}; const traits = fetchFinalSetOfTraits(message); @@ -217,7 +219,7 @@ const getTransformedJSON = async (message, destination, propertyMap) => { if (!propertyMap) { // fetch HS properties // eslint-disable-next-line no-param-reassign - propertyMap = await getProperties(destination); + propertyMap = await getProperties(destination, metadata); } rawPayload = constructPayload(message, hsCommonConfigJson); @@ -325,7 +327,7 @@ const getLookupFieldValue = (message, lookupField) => { * @param {*} destination * @returns */ -const searchContacts = async (message, destination) => { +const searchContacts = async (message, destination, metadata) => { const { Config } = destination; let searchContactsResponse; let contactId; @@ -377,6 +379,7 @@ const searchContacts = async (message, destination) => { endpointPath, requestMethod: 'POST', module: 'router', + metadata, }, ); searchContactsResponse = processAxiosResponse(searchContactsResponse); @@ -389,6 +392,7 @@ const searchContacts = async (message, destination) => { endpointPath, requestMethod: 'POST', module: 'router', + metadata, }); searchContactsResponse = processAxiosResponse(searchContactsResponse); } @@ -528,6 +532,7 @@ const performHubSpotSearch = async ( objectType, identifierType, destination, + metadata, ) => { let checkAfter = 1; const searchResults = []; @@ -556,6 +561,7 @@ const performHubSpotSearch = async ( endpointPath, requestMethod: 'POST', module: 'router', + metadata, }); const processedResponse = processAxiosResponse(searchResponse); @@ -655,7 +661,7 @@ const getRequestData = (identifierType, chunk) => { * @param {*} inputs * @param {*} destination */ -const getExistingContactsData = async (inputs, destination) => { +const getExistingContactsData = async (inputs, destination, metadata) => { const { Config } = destination; const hsIdsToBeUpdated = []; const firstMessage = inputs[0].message; @@ -683,6 +689,7 @@ const getExistingContactsData = async (inputs, destination) => { objectType, identifierType, destination, + metadata, ); if (searchResults.length > 0) { hsIdsToBeUpdated.push(...searchResults); @@ -728,9 +735,9 @@ const setHsSearchId = (input, id, useSecondaryProp = false) => { * For email as primary key we use `hs_additional_emails` as well property to search existing contacts * */ -const splitEventsForCreateUpdate = async (inputs, destination) => { +const splitEventsForCreateUpdate = async (inputs, destination, metadata) => { // get all the id and properties of already existing objects needed for update. - const hsIdsToBeUpdated = await getExistingContactsData(inputs, destination); + const hsIdsToBeUpdated = await getExistingContactsData(inputs, destination, metadata); const resultInput = inputs.map((input) => { const { message } = input; @@ -805,12 +812,12 @@ const getHsSearchId = (message) => { * @param {*} traits * @param {*} destination */ -const populateTraits = async (propertyMap, traits, destination) => { +const populateTraits = async (propertyMap, traits, destination, metadata) => { const populatedTraits = traits; let propertyToTypeMap = propertyMap; if (!propertyToTypeMap) { // fetch HS properties - propertyToTypeMap = await getProperties(destination); + propertyToTypeMap = await getProperties(destination, metadata); } const keys = Object.keys(populatedTraits); diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 4e61d65982..53d82158c5 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -46,12 +46,12 @@ const getIdFromNewOrExistingProfile = async ({ endpoint, payload, requestOptions payload, requestOptions, { + metadata, destType: 'klaviyo', feature: 'transformation', endpointPath, requestMethod: 'POST', module: 'router', - metadata, }, ); diff --git a/src/v0/destinations/marketo/transform.js b/src/v0/destinations/marketo/transform.js index b811596f95..accca1d449 100644 --- a/src/v0/destinations/marketo/transform.js +++ b/src/v0/destinations/marketo/transform.js @@ -55,7 +55,7 @@ const authCache = new Cache(AUTH_CACHE_TTL); // 1 hr // fails the transformer if auth fails // ------------------------ // Ref: https://developers.marketo.com/rest-api/authentication/#creating_an_access_token -const getAuthToken = async (formattedDestination) => +const getAuthToken = async (formattedDestination, metadata) => authCache.get(formattedDestination.ID, async () => { const { accountId, clientId, clientSecret } = formattedDestination; const clientResponse = await sendGetRequest( @@ -67,6 +67,7 @@ const getAuthToken = async (formattedDestination) => grant_type: 'client_credentials', }, }, + metadata, ); const data = marketoResponseHandler(clientResponse, 'During fetching auth token'); if (data) { @@ -92,7 +93,7 @@ const getAuthToken = async (formattedDestination) => // If lookupField is omitted, the default key is email. // ------------------------ // Thus we'll always be using createOrUpdate -const createOrUpdateLead = async (formattedDestination, token, userId, anonymousId) => +const createOrUpdateLead = async (formattedDestination, token, userId, anonymousId, metadata) => userIdLeadCache.get(userId || anonymousId, async () => { const attribute = userId ? { userId } : { anonymousId }; stats.increment(LEAD_LOOKUP_METRIC, { @@ -114,6 +115,7 @@ const createOrUpdateLead = async (formattedDestination, token, userId, anonymous 'Content-type': JSON_MIME_TYPE, }, }, + metadata, ); const data = getResponseHandlerData( clientResponse, @@ -135,7 +137,7 @@ const createOrUpdateLead = async (formattedDestination, token, userId, anonymous // ------------------------ // Ref: https://developers.marketo.com/rest-api/lead-database/leads/#create_and_update // ------------------------ -const lookupLeadUsingEmail = async (formattedDestination, token, email) => +const lookupLeadUsingEmail = async (formattedDestination, token, email, metadata) => emailLeadCache.get(email, async () => { stats.increment(LEAD_LOOKUP_METRIC, { type: 'email', action: 'fetch' }); const clientResponse = await sendGetRequest( @@ -145,6 +147,7 @@ const lookupLeadUsingEmail = async (formattedDestination, token, email) => params: { filterValues: email, filterType: 'email' }, headers: { Authorization: `Bearer ${token}` }, }, + metadata, ); const data = getResponseHandlerData( clientResponse, @@ -167,7 +170,7 @@ const lookupLeadUsingEmail = async (formattedDestination, token, email) => // ------------------------ // Ref: https://developers.marketo.com/rest-api/lead-database/leads/#create_and_update // ------------------------ -const lookupLeadUsingId = async (formattedDestination, token, userId, anonymousId) => +const lookupLeadUsingId = async (formattedDestination, token, userId, anonymousId, metadata) => userIdLeadCache.get(userId || anonymousId, async () => { stats.increment(LEAD_LOOKUP_METRIC, { type: 'userId', action: 'fetch' }); const clientResponse = await sendGetRequest( @@ -179,6 +182,7 @@ const lookupLeadUsingId = async (formattedDestination, token, userId, anonymousI }, headers: { Authorization: `Bearer ${token}` }, }, + metadata, ); const data = getResponseHandlerData( clientResponse, @@ -195,7 +199,7 @@ const lookupLeadUsingId = async (formattedDestination, token, userId, anonymousI return null; }); -const getLeadId = async (message, formattedDestination, token) => { +const getLeadId = async (message, formattedDestination, token, metadata) => { // precedence ->> // -> externalId (context.externalId[0].type == marketoLeadId) // -> lookup lead using email @@ -225,10 +229,16 @@ const getLeadId = async (message, formattedDestination, token) => { if (!leadId) { // search for lead using email if (email) { - leadId = await lookupLeadUsingEmail(formattedDestination, token, email); + leadId = await lookupLeadUsingEmail(formattedDestination, token, email, metadata); } else { // search lead using userId or anonymousId - leadId = await lookupLeadUsingId(formattedDestination, token, userId, message.anonymousId); + leadId = await lookupLeadUsingId( + formattedDestination, + token, + userId, + message.anonymousId, + metadata, + ); } } @@ -236,7 +246,13 @@ const getLeadId = async (message, formattedDestination, token) => { if (!leadId) { // check we have permission to create lead on marketo if (formattedDestination.createIfNotExist) { - leadId = await createOrUpdateLead(formattedDestination, token, userId, message.anonymousId); + leadId = await createOrUpdateLead( + formattedDestination, + token, + userId, + message.anonymousId, + metadata, + ); } else { throw new ConfigurationError('Lead creation is turned off on the dashboard'); } @@ -264,7 +280,7 @@ const getLeadId = async (message, formattedDestination, token) => { // ------------------------ // Almost same as leadId lookup. Noticable difference from lookup is we'll using // `id` i.e. leadId as lookupField at the end of it -const processIdentify = async (message, formattedDestination, token) => { +const processIdentify = async (message, formattedDestination, token, metadata) => { // If mapped to destination, Add externalId to traits if (get(message, MappedToDestinationKey)) { addExternalIdToTraits(message); @@ -277,7 +293,7 @@ const processIdentify = async (message, formattedDestination, token) => { throw new InstrumentationError('Invalid traits value for Marketo'); } - const leadId = await getLeadId(message, formattedDestination, token); + const leadId = await getLeadId(message, formattedDestination, token, metadata); let attribute = constructPayload(traits, identifyConfig); // leadTraitMapping will not be used if mapping is done through VDM in rETL @@ -338,7 +354,7 @@ const processIdentify = async (message, formattedDestination, token) => { // process track events - only mapped events // ------------------------ // Ref: https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/addCustomActivityUsingPOST -const processTrack = async (message, formattedDestination, token) => { +const processTrack = async (message, formattedDestination, token, metadata) => { // check if trackAnonymousEvent is turned off and userId is not present - fail // check if the event is mapped in customActivityEventMap. if not - fail // get primaryKey name for the event @@ -373,7 +389,7 @@ const processTrack = async (message, formattedDestination, token) => { } // get leadId - const leadId = await getLeadId(message, formattedDestination, token); + const leadId = await getLeadId(message, formattedDestination, token, metadata); // handle addition of custom activity attributes // Reference: https://developers.marketo.com/rest-api/lead-database/activities/#add_custom_activities @@ -420,7 +436,7 @@ const responseWrapper = (response) => { return resp; }; -const processEvent = async (message, destination, token) => { +const processEvent = async ({ message, destination, metadata }, token) => { if (!message.type) { throw new InstrumentationError('Message Type is not present. Aborting message.'); } @@ -430,10 +446,10 @@ const processEvent = async (message, destination, token) => { let response; switch (messageType) { case EventType.IDENTIFY: - response = await processIdentify(message, formattedDestination, token); + response = await processIdentify(message, formattedDestination, token, metadata); break; case EventType.TRACK: - response = await processTrack(message, formattedDestination, token); + response = await processTrack(message, formattedDestination, token, metadata); break; default: throw new InstrumentationError('Message type not supported'); @@ -444,11 +460,11 @@ const processEvent = async (message, destination, token) => { }; const process = async (event) => { - const token = await getAuthToken(formatConfig(event.destination)); + const token = await getAuthToken(formatConfig(event.destination), event.metadata); if (!token) { throw new UnauthorizedError('Authorization failed'); } - const response = await processEvent(event.message, event.destination, token); + const response = await processEvent(event, token); return response; }; @@ -457,7 +473,7 @@ const processRouterDest = async (inputs, reqMetadata) => { // If destination information is not present Error should be thrown let token; try { - token = await getAuthToken(formatConfig(inputs[0].destination)); + token = await getAuthToken(formatConfig(inputs[0].destination), inputs[0].metadata); // If token is null track/identify calls cannot be executed. if (!token) { @@ -483,7 +499,7 @@ const processRouterDest = async (inputs, reqMetadata) => { inputs.map(async (input) => { try { return getSuccessRespEvents( - await processEvent(input.message, input.destination, token), + await processEvent(input, token), [input.metadata], input.destination, ); diff --git a/src/v0/destinations/marketo/util.js b/src/v0/destinations/marketo/util.js index b3a24fb411..aee872efac 100644 --- a/src/v0/destinations/marketo/util.js +++ b/src/v0/destinations/marketo/util.js @@ -243,13 +243,14 @@ const marketoResponseHandler = ( * @param {*} options * @returns { response, status } */ -const sendGetRequest = async (url, options) => { +const sendGetRequest = async (url, options, metadata) => { const clientResponse = await httpGET(url, options, { destType: 'marketo', feature: 'transformation', endpointPath: `/v1/leads`, requestMethod: 'GET', module: 'router', + metadata, }); const processedResponse = processAxiosResponse(clientResponse); return processedResponse; @@ -261,13 +262,14 @@ const sendGetRequest = async (url, options) => { * @param {*} options * @returns { response, status } */ -const sendPostRequest = async (url, data, options) => { +const sendPostRequest = async (url, data, options, metadata) => { const clientResponse = await httpPOST(url, data, options, { destType: 'marketo', feature: 'transformation', endpointPath: `/v1/leads`, requestMethod: 'POST', module: 'router', + metadata, }); const processedResponse = processAxiosResponse(clientResponse); return processedResponse; diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index 92c137c614..810b528bbf 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -93,7 +93,7 @@ const processEvent = (event) => { // eslint-disable-next-line @typescript-eslint/no-unused-vars const process = async (event, _processParams) => { - const token = await getAuthToken(formatConfig(event.destination)); + const token = await getAuthToken(formatConfig(event.destination), event.metadata); if (!token) { throw new UnauthorizedError('Authorization failed'); } @@ -106,9 +106,9 @@ const processRouterDest = async (inputs, reqMetadata) => { // Token needs to be generated for marketo which will be done on input level. // If destination information is not present Error should be thrown - const { destination } = inputs[0]; + const { destination, metadata } = inputs[0]; try { - const token = await getAuthToken(formatConfig(destination)); + const token = await getAuthToken(formatConfig(destination), metadata); if (!token) { throw new UnauthorizedError('Could not retrieve authorisation token'); } diff --git a/src/v0/destinations/mautic/transform.js b/src/v0/destinations/mautic/transform.js index 13808f6e3c..56cfceb371 100644 --- a/src/v0/destinations/mautic/transform.js +++ b/src/v0/destinations/mautic/transform.js @@ -56,7 +56,7 @@ const responseBuilder = async (payload, endpoint, method, messageType, Config) = * @param {*} endPoint * @returns build response for group call */ -const groupResponseBuilder = async (message, Config, endPoint) => { +const groupResponseBuilder = async ({ message, Config, metadata }, endPoint) => { let groupClass; validateGroupCall(message); switch (message.traits?.type?.toLowerCase()) { @@ -76,7 +76,7 @@ const groupResponseBuilder = async (message, Config, endPoint) => { } let contactId = getDestinationExternalID(message, 'mauticContactId'); if (!contactId) { - const contacts = await searchContactIds(message, Config, endPoint); + const contacts = await searchContactIds({ message, Config, metadata }, endPoint); if (!contacts || contacts.length === 0) { throw new ConfigurationError('Could not find any contact ID on lookup'); } @@ -117,7 +117,7 @@ const groupResponseBuilder = async (message, Config, endPoint) => { * @param {*} endPoint * @returns build response for identify call */ -const identifyResponseBuilder = async (message, Config, endpoint) => { +const identifyResponseBuilder = async ({ message, Config, metadata }, endpoint) => { let method; let endPoint; // constructing payload from mapping JSONs @@ -135,7 +135,7 @@ const identifyResponseBuilder = async (message, Config, endpoint) => { let contactId = getDestinationExternalID(message, 'mauticContactId'); if (!contactId) { - const contacts = await searchContactIds(message, Config, endpoint); + const contacts = await searchContactIds({ message, Config, metadata }, endpoint); if (contacts?.length === 1) { const [first] = contacts; contactId = first; @@ -154,7 +154,7 @@ const identifyResponseBuilder = async (message, Config, endpoint) => { }; const process = async (event) => { - const { message, destination } = event; + const { message, destination, metadata } = event; const { password, userName } = destination.Config; if (!password) { throw new ConfigurationError( @@ -178,10 +178,16 @@ const process = async (event) => { let response; switch (messageType) { case EventType.IDENTIFY: - response = await identifyResponseBuilder(message, destination.Config, endpoint); + response = await identifyResponseBuilder( + { message, Config: destination.Config, metadata }, + endpoint, + ); break; case EventType.GROUP: - response = await groupResponseBuilder(message, destination.Config, endpoint); + response = await groupResponseBuilder( + { message, Config: destination.Config, metadata }, + endpoint, + ); break; default: throw new InstrumentationError(`Event type "${messageType}" is not supported`); diff --git a/src/v0/destinations/mautic/utils.js b/src/v0/destinations/mautic/utils.js index 7a1827e769..fc9654d2e3 100644 --- a/src/v0/destinations/mautic/utils.js +++ b/src/v0/destinations/mautic/utils.js @@ -154,7 +154,7 @@ const validateGroupCall = (message) => { * It checks for lookUpfield Validation and make axios call ,if Valid, and returns the contactIDs received. * It Gets the contact Id using Lookup field and then email, otherwise returns null */ -const searchContactIds = async (message, Config, baseUrl) => { +const searchContactIds = async ({ message, Config, metadata }, baseUrl) => { const { lookUpField, userName, password } = Config; const traits = getFieldValueFromMessage(message, 'traits'); @@ -186,6 +186,7 @@ const searchContactIds = async (message, Config, baseUrl) => { endpointPath: '/contacts', requestMethod: 'GET', module: 'router', + metadata, }, ); searchContactsResponse = processAxiosResponse(searchContactsResponse); diff --git a/src/v0/destinations/monday/transform.js b/src/v0/destinations/monday/transform.js index 152b42f8d0..de34fa0521 100644 --- a/src/v0/destinations/monday/transform.js +++ b/src/v0/destinations/monday/transform.js @@ -38,7 +38,7 @@ const responseBuilder = (payload, endpoint, apiToken) => { * @param {*} param1 * @returns */ -const trackResponseBuilder = async (message, { Config }) => { +const trackResponseBuilder = async ({ message, destination: { Config }, metadata }) => { const { apiToken } = Config; let boardId = getDestinationExternalID(message, 'boardId'); const event = get(message, 'event'); @@ -54,14 +54,14 @@ const trackResponseBuilder = async (message, { Config }) => { } const endpoint = ENDPOINT; - const processedResponse = await getBoardDetails(endpoint, boardId, apiToken); + const processedResponse = await getBoardDetails(endpoint, boardId, apiToken, metadata); const payload = populatePayload(message, Config, processedResponse); return responseBuilder(payload, endpoint, apiToken); }; -const processEvent = async (message, destination) => { +const processEvent = async ({ message, destination, metadata }) => { if (!message.type) { throw new InstrumentationError('Event type is required'); } @@ -71,14 +71,14 @@ const processEvent = async (message, destination) => { const messageType = message.type.toLowerCase(); let response; if (messageType === EventType.TRACK) { - response = await trackResponseBuilder(message, destination); + response = await trackResponseBuilder({ message, destination, metadata }); } else { throw new InstrumentationError(`Event type ${messageType} is not supported`); } return response; }; -const process = async (event) => processEvent(event.message, event.destination); +const process = async (event) => processEvent(event); const processRouterDest = async (inputs, reqMetadata) => { const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); diff --git a/src/v0/destinations/monday/util.js b/src/v0/destinations/monday/util.js index 0694028eb2..07e084c158 100644 --- a/src/v0/destinations/monday/util.js +++ b/src/v0/destinations/monday/util.js @@ -179,7 +179,7 @@ const mapColumnValues = (properties, columnToPropertyMapping, board) => { * @param {*} apiToken * @returns */ -const getBoardDetails = async (url, boardID, apiToken) => { +const getBoardDetails = async (url, boardID, apiToken, metadata) => { const clientResponse = await httpPOST( url, { @@ -197,6 +197,7 @@ const getBoardDetails = async (url, boardID, apiToken) => { endpointPath: '/v2', requestMethod: 'POST', module: 'router', + metadata, }, ); const boardDetailsResponse = processAxiosResponse(clientResponse); diff --git a/src/v0/destinations/pardot/networkHandler.js b/src/v0/destinations/pardot/networkHandler.js index 60d2f7ee23..6344301d39 100644 --- a/src/v0/destinations/pardot/networkHandler.js +++ b/src/v0/destinations/pardot/networkHandler.js @@ -118,6 +118,7 @@ const prepareProxyReq = (request) => { * @returns */ const pardotProxyRequest = async (request) => { + const { metadata } = request; const { endpoint, data, method, params, headers } = prepareProxyReq(request); const requestOptions = { @@ -130,6 +131,7 @@ const pardotProxyRequest = async (request) => { const response = await httpSend(requestOptions, { feature: 'proxy', destType: 'pardot', + metadata, }); return response; }; diff --git a/src/v0/destinations/profitwell/transform.js b/src/v0/destinations/profitwell/transform.js index 58449fd9c1..e88718771e 100644 --- a/src/v0/destinations/profitwell/transform.js +++ b/src/v0/destinations/profitwell/transform.js @@ -24,18 +24,19 @@ const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); const tags = require('../../util/tags'); const { JSON_MIME_TYPE } = require('../../util/constant'); -const identifyResponseBuilder = async (message, { Config }) => { +const identifyResponseBuilder = async ({ message, destination: { Config }, metadata }) => { const { userId, userAlias, subscriptionId, subscriptionAlias } = validatePayloadAndRetunImpIds(message); let finalSubscriptionId = subscriptionId; let finalSubscriptionAlias = subscriptionAlias; - - const targetUrl = `${BASE_ENDPOINT}/v2/users/${userId || userAlias}/`; - const res = await getSubscriptionHistory(targetUrl, { + const options = { headers: { Authorization: Config.privateApiKey, }, - }); + }; + + const targetUrl = `${BASE_ENDPOINT}/v2/users/${userId || userAlias}/`; + const res = await getSubscriptionHistory(targetUrl, options, metadata); let payload; const response = defaultRequestConfig(); @@ -159,7 +160,7 @@ const process = async (event) => { let response; if (messageType === EventType.IDENTIFY) { - response = await identifyResponseBuilder(message, destination); + response = await identifyResponseBuilder(event); } else { throw new InstrumentationError(`Event type ${messageType} is not supported`); } diff --git a/src/v0/destinations/profitwell/utils.js b/src/v0/destinations/profitwell/utils.js index 1b23561721..cdc4f8a47d 100644 --- a/src/v0/destinations/profitwell/utils.js +++ b/src/v0/destinations/profitwell/utils.js @@ -179,7 +179,7 @@ const CURRENCY_CODES = [ 'zwl', ]; -const getSubscriptionHistory = async (endpoint, options) => { +const getSubscriptionHistory = async (endpoint, options, metadata) => { const requestOptions = { method: 'get', ...options, @@ -191,6 +191,7 @@ const getSubscriptionHistory = async (endpoint, options) => { endpointPath: '/users/userId', requestMethod: 'GET', module: 'router', + metadata, }); return res; }; diff --git a/src/v0/destinations/rakuten/networkHandler.js b/src/v0/destinations/rakuten/networkHandler.js index 4c97a23e51..95189cab62 100644 --- a/src/v0/destinations/rakuten/networkHandler.js +++ b/src/v0/destinations/rakuten/networkHandler.js @@ -10,6 +10,7 @@ const { HTTP_STATUS_CODES } = require('../../util/constant'); const DESTINATION = 'RAKUTEN'; const prepareProxyRequest = (request) => request; const proxyRequest = async (request, destType) => { + const { metadata } = request; const { endpoint, data, method, params, headers } = prepareProxyRequest(request); const requestOptions = { url: endpoint, @@ -21,6 +22,7 @@ const proxyRequest = async (request, destType) => { const response = await httpSend(requestOptions, { feature: 'proxy', destType, + metadata, endpointPath: '/ep', requestMethod: 'GET', module: 'dataDelivery', diff --git a/src/v0/destinations/salesforce/transform.js b/src/v0/destinations/salesforce/transform.js index b8f032c5bf..9b7123c207 100644 --- a/src/v0/destinations/salesforce/transform.js +++ b/src/v0/destinations/salesforce/transform.js @@ -106,7 +106,7 @@ async function getSaleforceIdForRecord( objectType, identifierType, identifierValue, - destination, + { destination, metadata }, authorizationFlow, ) { const objSearchUrl = `${authorizationData.instanceUrl}/services/data/v${SF_API_VERSION}/parameterizedSearch/?q=${identifierValue}&sobject=${objectType}&in=${identifierType}&${objectType}.fields=id,${identifierType}`; @@ -117,6 +117,7 @@ async function getSaleforceIdForRecord( headers: getAuthHeader({ authorizationFlow, authorizationData }), }, { + metadata, destType: 'salesforce', feature: 'transformation', endpointPath: '/parameterizedSearch', @@ -156,9 +157,8 @@ async function getSaleforceIdForRecord( // // Default Object type will be "Lead" for backward compatibility async function getSalesforceIdFromPayload( - message, + { message, destination, metadata }, authorizationData, - destination, authorizationFlow, ) { // define default map @@ -201,7 +201,7 @@ async function getSalesforceIdFromPayload( objectType, identifierType, id, - destination, + { destination, metadata }, authorizationFlow, ); } @@ -233,6 +233,7 @@ async function getSalesforceIdFromPayload( headers: getAuthHeader({ authorizationFlow, authorizationData }), }, { + metadata, destType: 'salesforce', feature: 'transformation', endpointPath: '/parameterizedSearch', @@ -283,7 +284,11 @@ async function getSalesforceIdFromPayload( } // Function for handling identify events -async function processIdentify(message, authorizationData, destination, authorizationFlow) { +async function processIdentify( + { message, destination, metadata }, + authorizationData, + authorizationFlow, +) { const mapProperty = destination.Config.mapProperty === undefined ? true : destination.Config.mapProperty; // check the traits before hand @@ -305,9 +310,8 @@ async function processIdentify(message, authorizationData, destination, authoriz // get salesforce object map const salesforceMaps = await getSalesforceIdFromPayload( - message, + { message, destination, metadata }, authorizationData, - destination, authorizationFlow, ); @@ -331,10 +335,18 @@ async function processIdentify(message, authorizationData, destination, authoriz // Generic process function which invokes specific handler functions depending on message type // and event type where applicable -async function processSingleMessage(message, authorizationData, destination, authorizationFlow) { +async function processSingleMessage( + { message, destination, metadata }, + authorizationData, + authorizationFlow, +) { let response; if (message.type === EventType.IDENTIFY) { - response = await processIdentify(message, authorizationData, destination, authorizationFlow); + response = await processIdentify( + { message, destination, metadata }, + authorizationData, + authorizationFlow, + ); } else { throw new InstrumentationError(`message type ${message.type} is not supported`); } @@ -344,9 +356,8 @@ async function processSingleMessage(message, authorizationData, destination, aut async function process(event) { const authInfo = await collectAuthorizationInfo(event); const response = await processSingleMessage( - event.message, + event, authInfo.authorizationData, - event.destination, authInfo.authorizationFlow, ); return response; @@ -377,12 +388,7 @@ const processRouterDest = async (inputs, reqMetadata) => { // unprocessed payload return getSuccessRespEvents( - await processSingleMessage( - input.message, - authInfo.authorizationData, - input.destination, - authInfo.authorizationFlow, - ), + await processSingleMessage(input, authInfo.authorizationData, authInfo.authorizationFlow), [input.metadata], input.destination, ); diff --git a/src/v0/destinations/salesforce/utils.js b/src/v0/destinations/salesforce/utils.js index bb1236d290..9a4effc502 100644 --- a/src/v0/destinations/salesforce/utils.js +++ b/src/v0/destinations/salesforce/utils.js @@ -101,7 +101,7 @@ const salesforceResponseHandler = (destResponse, sourceMessage, authKey, authori * Utility method to construct the header to be used for SFDC API calls * The "Authorization: Bearer " header element needs to be passed * for authentication for all SFDC REST API calls - * @param {*} destination + * @param {destination: Record, metadata: Record} * @returns */ const getAccessTokenOauth = (metadata) => ({ @@ -109,7 +109,7 @@ const getAccessTokenOauth = (metadata) => ({ instanceUrl: metadata.secret?.instance_url, }); -const getAccessToken = async (destination) => { +const getAccessToken = async ({ destination, metadata }) => { const accessTokenKey = destination.ID; return ACCESS_TOKEN_CACHE.get(accessTokenKey, async () => { @@ -137,6 +137,7 @@ const getAccessToken = async (destination) => { endpointPath: '/services/oauth2/token', requestMethod: 'POST', module: 'router', + metadata, }, ); // If the request fails, throwing error. @@ -173,7 +174,7 @@ const collectAuthorizationInfo = async (event) => { authorizationData = getAccessTokenOauth(event.metadata); } else { authorizationFlow = LEGACY; - authorizationData = await getAccessToken(event.destination); + authorizationData = await getAccessToken(event); } return { authorizationFlow, authorizationData }; }; diff --git a/src/v0/destinations/sendgrid/transform.js b/src/v0/destinations/sendgrid/transform.js index c32e34c489..a3516687db 100644 --- a/src/v0/destinations/sendgrid/transform.js +++ b/src/v0/destinations/sendgrid/transform.js @@ -57,15 +57,15 @@ const responseBuilder = (payload, method, endpoint, apiKey) => { throw new TransformationError(ErrorMessage.FailedToConstructPayload); }; -const identifyResponseBuilder = async (message, destination) => { +const identifyResponseBuilder = async ({ message, destination, metadata }) => { validateIdentifyPayload(message); - const builder = await createOrUpdateContactPayloadBuilder(message, destination); + const builder = await createOrUpdateContactPayloadBuilder({ message, destination, metadata }); const { payload, method, endpoint } = builder; const { apiKey } = destination.Config; return responseBuilder(payload, method, endpoint, apiKey); }; -const trackResponseBuilder = async (message, { Config }) => { +const trackResponseBuilder = async ({ message, destination: { Config } }) => { validateTrackPayload(message, Config); let payload = {}; payload = constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK.name]); @@ -123,7 +123,8 @@ const trackResponseBuilder = async (message, { Config }) => { return responseBuilder(payload, method, endpoint, apiKey); }; -const processEvent = async (message, destination) => { +const processEvent = async (event) => { + const { message, destination } = event; // Validating if message type is even given or not if (!message.type) { throw new InstrumentationError('Event type is required'); @@ -137,10 +138,10 @@ const processEvent = async (message, destination) => { let response; switch (messageType) { case EventType.IDENTIFY: - response = await identifyResponseBuilder(message, destination); + response = await identifyResponseBuilder(event); break; case EventType.TRACK: - response = await trackResponseBuilder(message, destination); + response = await trackResponseBuilder(event); break; default: throw new InstrumentationError(`Event type ${messageType} is not supported`); @@ -148,7 +149,7 @@ const processEvent = async (message, destination) => { return response; }; -const process = (event) => processEvent(event.message, event.destination); +const process = (event) => processEvent(event); const generateBatchedPaylaodForArray = (events, combination) => { let batchEventResponse = defaultBatchRequestConfig(); diff --git a/src/v0/destinations/sendgrid/util.js b/src/v0/destinations/sendgrid/util.js index 7105c5cda5..1edb480516 100644 --- a/src/v0/destinations/sendgrid/util.js +++ b/src/v0/destinations/sendgrid/util.js @@ -431,7 +431,7 @@ const getContactListIds = (message, destination) => { * @param {*} destination * @returns */ -const fetchCustomFields = async (destination) => { +const fetchCustomFields = async ({ destination, metadata }) => { const { apiKey } = destination.Config; return customFieldsCache.get(destination.ID, async () => { const requestOptions = { @@ -448,6 +448,7 @@ const fetchCustomFields = async (destination) => { endpointPath: '/marketing/field_definitions', requestMethod: 'GET', module: 'router', + metadata, }); const processedResponse = processAxiosResponse(resonse); if (isHttpStatusSuccess(processedResponse.status)) { @@ -475,14 +476,14 @@ const fetchCustomFields = async (destination) => { * @param {*} contactDetails * @returns */ -const getCustomFields = async (message, destination) => { +const getCustomFields = async ({ message, destination, metadata }) => { const customFields = {}; const payload = get(message, 'context.traits'); const { customFieldsMapping } = destination.Config; const fieldsMapping = getHashFromArray(customFieldsMapping, 'from', 'to', false); const fields = Object.keys(fieldsMapping); if (fields.length > 0) { - const destinationCustomFields = await fetchCustomFields(destination); + const destinationCustomFields = await fetchCustomFields({ destination, metadata }); const customFieldNameToIdMapping = {}; const customFieldNamesArray = destinationCustomFields.map((destinationCustomField) => { const { id, name } = destinationCustomField; @@ -511,13 +512,13 @@ const getCustomFields = async (message, destination) => { * @param {*} destination * @returns */ -const createOrUpdateContactPayloadBuilder = async (message, destination) => { +const createOrUpdateContactPayloadBuilder = async ({ message, destination, metadata }) => { const contactDetails = constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.IDENTIFY.name]); if (contactDetails.address_line_1) { contactDetails.address_line_1 = flattenAddress(contactDetails.address_line_1); } const contactListIds = getContactListIds(message, destination); - contactDetails.custom_fields = await getCustomFields(message, destination); + contactDetails.custom_fields = await getCustomFields({ message, destination, metadata }); const payload = { contactDetails, contactListIds }; const { endpoint } = CONFIG_CATEGORIES.IDENTIFY; const method = defaultPutRequestConfig.requestMethod; diff --git a/src/v0/destinations/sendinblue/transform.js b/src/v0/destinations/sendinblue/transform.js index 9514359d02..4e22200ee5 100644 --- a/src/v0/destinations/sendinblue/transform.js +++ b/src/v0/destinations/sendinblue/transform.js @@ -176,7 +176,7 @@ const updateDOIContactResponseBuilder = (message, destination, identifier) => { ); }; -const createOrUpdateDOIContactResponseBuilder = async (message, destination) => { +const createOrUpdateDOIContactResponseBuilder = async ({ message, destination, metadata }) => { let email = getFieldValueFromMessage(message, 'emailOnly'); const phone = getFieldValueFromMessage(message, 'phone'); @@ -196,7 +196,7 @@ const createOrUpdateDOIContactResponseBuilder = async (message, destination) => } const { apiKey } = destination.Config; - const contactExists = await checkIfContactExists(identifier, apiKey); + const contactExists = await checkIfContactExists(identifier, apiKey, metadata); if (contactExists) { return updateDOIContactResponseBuilder(message, destination, identifier); @@ -205,7 +205,7 @@ const createOrUpdateDOIContactResponseBuilder = async (message, destination) => return createDOIContactResponseBuilder(message, destination); }; -const identifyResponseBuilder = async (message, destination) => { +const identifyResponseBuilder = async ({ message, destination, metadata }) => { const { doi } = destination.Config; if (!doi) { const unlinkListIds = getListIds(message, 'sendinblueUnlinkListIds'); @@ -215,7 +215,7 @@ const identifyResponseBuilder = async (message, destination) => { return createOrUpdateContactResponseBuilder(message, destination); } - return createOrUpdateDOIContactResponseBuilder(message, destination); + return createOrUpdateDOIContactResponseBuilder({ message, destination, metadata }); }; // ref:- https://tracker-doc.sendinblue.com/reference/trackevent-3 @@ -305,7 +305,8 @@ const pageResponseBuilder = (message, destination) => { return responseBuilder(payload, endpoint, destination, true); }; -const processEvent = async (message, destination) => { +const processEvent = async (event) => { + const { message, destination } = event; if (!message.type) { throw new InstrumentationError('Event type is required'); } @@ -314,7 +315,7 @@ const processEvent = async (message, destination) => { let response; switch (messageType) { case EventType.IDENTIFY: - response = await identifyResponseBuilder(message, destination); + response = await identifyResponseBuilder(event); break; case EventType.TRACK: response = trackResponseBuilder(message, destination); @@ -328,7 +329,7 @@ const processEvent = async (message, destination) => { return response; }; -const process = (event) => processEvent(event.message, event.destination); +const process = (event) => processEvent(event); const processRouterDest = async (inputs) => { const respList = await simpleProcessRouterDest(inputs, process, process); diff --git a/src/v0/destinations/sendinblue/util.js b/src/v0/destinations/sendinblue/util.js index e4862ccc39..3af35cd7b9 100644 --- a/src/v0/destinations/sendinblue/util.js +++ b/src/v0/destinations/sendinblue/util.js @@ -52,7 +52,7 @@ const validateEmailAndPhone = (email, phone = null) => { */ const prepareEmailFromPhone = (phone) => `${phone.replace('+', '')}${EMAIL_SUFFIX}`; -const checkIfContactExists = async (identifier, apiKey) => { +const checkIfContactExists = async (identifier, apiKey, metadata) => { const endpoint = getContactDetailsEndpoint(identifier); const requestOptions = { headers: prepareHeader(apiKey), @@ -63,6 +63,7 @@ const checkIfContactExists = async (identifier, apiKey) => { endpointPath: '/contacts', requestMethod: 'GET', module: 'router', + metadata, }); const processedContactDetailsResponse = processAxiosResponse(contactDetailsResponse); diff --git a/src/v0/destinations/sfmc/transform.js b/src/v0/destinations/sfmc/transform.js index bf474ff3f0..a433179f9c 100644 --- a/src/v0/destinations/sfmc/transform.js +++ b/src/v0/destinations/sfmc/transform.js @@ -31,7 +31,7 @@ const CONTACT_KEY_KEY = 'Contact Key'; // DOC: https://developer.salesforce.com/docs/atlas.en-us.mc-app-development.meta/mc-app-development/access-token-s2s.htm -const getToken = async (clientId, clientSecret, subdomain) => { +const getToken = async (clientId, clientSecret, subdomain, metadata) => { const { processedResponse: processedResponseSfmc } = await handleHttpRequest( 'post', `https://${subdomain}.${ENDPOINTS.GET_TOKEN}`, @@ -49,6 +49,7 @@ const getToken = async (clientId, clientSecret, subdomain) => { endpointPath: '/token', requestMethod: 'POST', module: 'router', + metadata, }, ); @@ -194,7 +195,7 @@ const responseBuilderForMessageEvent = (message, subDomain, authToken, hashMapEv return response; }; -const responseBuilderSimple = async (message, category, destination) => { +const responseBuilderSimple = async ({ message, destination, metadata }, category) => { const { clientId, clientSecret, @@ -213,7 +214,7 @@ const responseBuilderSimple = async (message, category, destination) => { // map from an event name to uuid as true or false to determine to send uuid as primary key or not. const hashMapUUID = getHashFromArray(eventToUUID, 'event', 'uuid'); // token needed for authorization for subsequent calls - const authToken = await getToken(clientId, clientSecret, subDomain); + const authToken = await getToken(clientId, clientSecret, subDomain, metadata); // map from an event name to an event definition key. const hashMapEventDefinition = getHashFromArray(eventToDefinitionMapping, 'from', 'to'); // if createOrUpdateContacts is true identify calls for create and update of contacts will not occur. @@ -270,7 +271,7 @@ const responseBuilderSimple = async (message, category, destination) => { throw new ConfigurationError(`Event type '${category.type}' not supported`); }; -const processEvent = async (message, destination) => { +const processEvent = async ({ message, destination, metadata }) => { if (!message.type) { throw new InstrumentationError('Event type is required'); } @@ -290,12 +291,12 @@ const processEvent = async (message, destination) => { } // build the response - const response = await responseBuilderSimple(message, category, destination); + const response = await responseBuilderSimple({ message, destination, metadata }, category); return response; }; const process = async (event) => { - const response = await processEvent(event.message, event.destination); + const response = await processEvent(event); return response; }; diff --git a/src/v0/destinations/sfmc/transform.test.js b/src/v0/destinations/sfmc/transform.test.js index 8d382ef649..e182fb0d78 100644 --- a/src/v0/destinations/sfmc/transform.test.js +++ b/src/v0/destinations/sfmc/transform.test.js @@ -33,7 +33,7 @@ describe('responseBuilderSimple', () => { name: 'Identify', }; - const response = await responseBuilderSimple(message, category, destination); + const response = await responseBuilderSimple({ message, destination }, category); expect(response).toHaveLength(2); expect(response[0]).toHaveProperty('endpoint'); @@ -58,7 +58,7 @@ describe('responseBuilderSimple', () => { }; try { - await responseBuilderSimple(message, category, destination); + await responseBuilderSimple({ message, destination }, category); } catch (e) { expect(e).toBeInstanceOf(ConfigurationError); expect(e.message).toBe('Event name is required for track events'); @@ -77,7 +77,7 @@ describe('responseBuilderSimple', () => { name: 'Track', }; try { - await responseBuilderSimple(message, category, destination); + await responseBuilderSimple({ message, destination }, category); } catch (e) { expect(e).toBeInstanceOf(ConfigurationError); expect(e.message).toBe('Event not mapped for this track call'); @@ -96,7 +96,7 @@ describe('responseBuilderSimple', () => { }; try { - await responseBuilderSimple(message, category, destination); + await responseBuilderSimple({ message, destination }, category); } catch (e) { expect(e).toBeInstanceOf(ConfigurationError); expect(e.message).toBe("Event type 'unsupported' not supported"); @@ -116,7 +116,7 @@ describe('responseBuilderSimple', () => { name: 'Track', }; - const response = await responseBuilderSimple(message, category, destination); + const response = await responseBuilderSimple({ message, destination }, category); expect(response).toHaveProperty('endpoint'); expect(response).toHaveProperty('method'); expect(response).toHaveProperty('body.JSON'); diff --git a/src/v0/destinations/snapchat_custom_audience/networkHandler.js b/src/v0/destinations/snapchat_custom_audience/networkHandler.js index 6044216293..da2a021345 100644 --- a/src/v0/destinations/snapchat_custom_audience/networkHandler.js +++ b/src/v0/destinations/snapchat_custom_audience/networkHandler.js @@ -31,6 +31,7 @@ const prepareProxyReq = (request) => { }; const scAudienceProxyRequest = async (request) => { + const { metadata } = request; const { endpoint, data, method, params, headers } = prepareProxyReq(request); const requestOptions = { @@ -46,6 +47,7 @@ const scAudienceProxyRequest = async (request) => { endpointPath: '/segments/segmentId/users', requestMethod: requestOptions?.method, module: 'dataDelivery', + metadata, }); return response; }; diff --git a/src/v0/destinations/the_trade_desk/networkHandler.js b/src/v0/destinations/the_trade_desk/networkHandler.js index aebbfc0785..d04b5216b0 100644 --- a/src/v0/destinations/the_trade_desk/networkHandler.js +++ b/src/v0/destinations/the_trade_desk/networkHandler.js @@ -10,6 +10,7 @@ const tags = require('../../util/tags'); const { JSON_MIME_TYPE } = require('../../util/constant'); const proxyRequest = async (request) => { + const { metadata } = request; const { endpoint, data, method, params, headers, config } = prepareProxyRequest(request); if (!config?.advertiserSecretKey) { @@ -43,6 +44,7 @@ const proxyRequest = async (request) => { endpointPath: '/track/realtimeconversion', requestMethod: 'POST', module: 'dataDelivery', + metadata, }); return response; }; diff --git a/src/v0/destinations/user/transform.js b/src/v0/destinations/user/transform.js index ed04f5ccd4..24baadd200 100644 --- a/src/v0/destinations/user/transform.js +++ b/src/v0/destinations/user/transform.js @@ -43,23 +43,24 @@ const responseBuilder = async (payload, endpoint, method, apiKey) => { throw new TransformationError('Something went wrong while constructing the payload'); }; -const identifyResponseBuilder = async (message, destination) => { +const identifyResponseBuilder = async (event) => { + const { destination } = event; let builder; - const user = await retrieveUserFromLookup(message, destination); + const user = await retrieveUserFromLookup(event); const { Config } = destination; const { apiKey } = Config; // If user already exist we will update it else creates a new user if (!user) { - builder = createOrUpdateUserPayloadBuilder(message, destination); + builder = createOrUpdateUserPayloadBuilder(event); } else { const { id } = user; - builder = createOrUpdateUserPayloadBuilder(message, destination, id); + builder = createOrUpdateUserPayloadBuilder(event, id); } const { payload, endpoint, method } = builder; return responseBuilder(payload, endpoint, method, apiKey); }; -const trackResponseBuilder = async (message, destination) => { +const trackResponseBuilder = async ({ message, destination, metadata }) => { if (!message.event) { throw new InstrumentationError('Parameter event is required'); } @@ -68,7 +69,7 @@ const trackResponseBuilder = async (message, destination) => { let endpoint; let method; let builder; - const user = await retrieveUserFromLookup(message, destination); + const user = await retrieveUserFromLookup({ message, destination, metadata }); const { Config } = destination; const { apiKey, appSubdomain } = Config; if (user) { @@ -85,12 +86,12 @@ const trackResponseBuilder = async (message, destination) => { ); }; -const pageResponseBuilder = async (message, destination) => { +const pageResponseBuilder = async ({ message, destination, metadata }) => { let payload; let endpoint; let method; let builder; - const user = await retrieveUserFromLookup(message, destination); + const user = await retrieveUserFromLookup({ message, destination, metadata }); const { Config } = destination; const { apiKey, appSubdomain } = Config; if (user) { @@ -106,14 +107,14 @@ const pageResponseBuilder = async (message, destination) => { ); }; -const groupResponseBuilder = async (message, destination) => { +const groupResponseBuilder = async ({ message, destination, metadata }) => { validateGroupPayload(message); let payload; let endpoint; let method; let builder; - const user = await getUserByCustomId(message, destination); + const user = await getUserByCustomId(message, destination, metadata); const { Config } = destination; const { apiKey, appSubdomain } = Config; /* @@ -121,11 +122,11 @@ const groupResponseBuilder = async (message, destination) => { * user does not exist -> throw an error */ if (user) { - let company = await getCompanyByCustomId(message, destination); + let company = await getCompanyByCustomId(message, destination, metadata); if (!company) { - company = await createCompany(message, destination); + company = await createCompany(message, destination, metadata); } else { - company = await updateCompany(message, destination, company); + company = await updateCompany(message, destination, company, metadata); } builder = addUserToCompanyPayloadBuilder(user, company); payload = builder.payload; @@ -137,7 +138,8 @@ const groupResponseBuilder = async (message, destination) => { throw new NetworkInstrumentationError('No user found with given userId'); }; -const processEvent = async (message, destination) => { +const processEvent = async (event) => { + const { message } = event; // Validating if message type is even given or not if (!message.type) { throw new InstrumentationError('Event type is required'); @@ -146,16 +148,16 @@ const processEvent = async (message, destination) => { let response; switch (messageType) { case EventType.IDENTIFY: - response = await identifyResponseBuilder(message, destination); + response = await identifyResponseBuilder(event); break; case EventType.GROUP: - response = await groupResponseBuilder(message, destination); + response = await groupResponseBuilder(event); break; case EventType.TRACK: - response = await trackResponseBuilder(message, destination); + response = await trackResponseBuilder(event); break; case EventType.PAGE: - response = await pageResponseBuilder(message, destination); + response = await pageResponseBuilder(event); break; default: throw new InstrumentationError(`Event type ${messageType} is not supported`); @@ -163,7 +165,7 @@ const processEvent = async (message, destination) => { return response; }; -const process = async (event) => processEvent(event.message, event.destination); +const process = async (event) => processEvent(event); const processRouterDest = async (inputs, reqMetadata) => { const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); diff --git a/src/v0/destinations/user/utils.js b/src/v0/destinations/user/utils.js index f332d7a4a7..f469d123d8 100644 --- a/src/v0/destinations/user/utils.js +++ b/src/v0/destinations/user/utils.js @@ -211,7 +211,7 @@ const validateGroupPayload = (message) => { * @param {*} destination * @returns */ -const createCompany = async (message, destination) => { +const createCompany = async (message, destination, metadata) => { const commonCompanyPropertiesPayload = constructPayload( message, MAPPING_CONFIG[CONFIG_CATEGORIES.CREATE_COMPANY.name], @@ -240,6 +240,7 @@ const createCompany = async (message, destination) => { endpointPath: `/companies/`, requestMethod: 'POST', module: 'router', + metadata, }); const data = processAxiosResponse(response); return data.response; @@ -253,7 +254,7 @@ const createCompany = async (message, destination) => { * @param {*} company * @returns */ -const updateCompany = async (message, destination, company) => { +const updateCompany = async (message, destination, company, metadata) => { const commonCompanyPropertiesPayload = constructPayload( message, MAPPING_CONFIG[CONFIG_CATEGORIES.UPDATE_COMPANY.name], @@ -283,6 +284,7 @@ const updateCompany = async (message, destination, company) => { endpointPath: `/companies/`, requestMethod: 'PUT', module: 'router', + metadata, }); const data = processAxiosResponse(response); return data.response; @@ -296,7 +298,7 @@ const updateCompany = async (message, destination, company) => { * @param {*} appSubdomain * @returns */ -const getUserByUserKey = async (apiKey, userKey, appSubdomain) => { +const getUserByUserKey = async (apiKey, userKey, appSubdomain, metadata) => { const endpoint = prepareUrl(`${BASE_ENDPOINT}/users/search/?key=${userKey}`, appSubdomain); const requestOptions = { headers: { @@ -312,6 +314,7 @@ const getUserByUserKey = async (apiKey, userKey, appSubdomain) => { endpointPath: `/users/search`, requestMethod: 'GET', module: 'router', + metadata, }); const processedUserResponse = processAxiosResponse(userResponse); if (processedUserResponse.status === 200) { @@ -328,7 +331,7 @@ const getUserByUserKey = async (apiKey, userKey, appSubdomain) => { * @param {*} appSubdomain * @returns */ -const getUserByEmail = async (apiKey, email, appSubdomain) => { +const getUserByEmail = async (apiKey, email, appSubdomain, metadata) => { if (!email) { throw new InstrumentationError('Lookup field : email value is not present'); } @@ -348,6 +351,7 @@ const getUserByEmail = async (apiKey, email, appSubdomain) => { endpointPath: `/users/search/?email`, requestMethod: 'GET', module: 'router', + metadata, }); const processedUserResponse = processAxiosResponse(userResponse); @@ -366,7 +370,7 @@ const getUserByEmail = async (apiKey, email, appSubdomain) => { * @param {*} appSubdomain * @returns */ -const getUserByPhoneNumber = async (apiKey, phoneNumber, appSubdomain) => { +const getUserByPhoneNumber = async (apiKey, phoneNumber, appSubdomain, metadata) => { if (!phoneNumber) { throw new InstrumentationError('Lookup field : phone value is not present'); } @@ -389,6 +393,7 @@ const getUserByPhoneNumber = async (apiKey, phoneNumber, appSubdomain) => { endpointPath: `/users/search/?phone_number`, requestMethod: 'GET', module: 'router', + metadata, }); const processedUserResponse = processAxiosResponse(userResponse); @@ -415,7 +420,7 @@ const getUserByPhoneNumber = async (apiKey, phoneNumber, appSubdomain) => { * @param {*} destination * @returns */ -const getUserByCustomId = async (message, destination) => { +const getUserByCustomId = async (message, destination, metadata) => { const { Config } = destination; const { appSubdomain, apiKey } = Config; const userCustomId = getFieldValueFromMessage(message, 'userId'); @@ -436,6 +441,7 @@ const getUserByCustomId = async (message, destination) => { endpointPath: `/users-by-id/`, requestMethod: 'GET', module: 'router', + metadata, }); const processedUserResponse = processAxiosResponse(userResponse); @@ -453,7 +459,7 @@ const getUserByCustomId = async (message, destination) => { * @param {*} destination * @returns */ -const getCompanyByCustomId = async (message, destination) => { +const getCompanyByCustomId = async (message, destination, metadata) => { const { Config } = destination; const { appSubdomain, apiKey } = Config; const companyCustomId = getFieldValueFromMessage(message, 'groupId'); @@ -474,6 +480,7 @@ const getCompanyByCustomId = async (message, destination) => { endpointPath: `/companies-by-id/`, requestMethod: 'GET', module: 'router', + metadata, }); const processedUserResponse = processAxiosResponse(response); if (processedUserResponse.status === 200) { @@ -490,12 +497,12 @@ const getCompanyByCustomId = async (message, destination) => { * @param {*} destination * @returns */ -const retrieveUserFromLookup = async (message, destination) => { +const retrieveUserFromLookup = async ({ message, destination, metadata }) => { const { Config } = destination; const { appSubdomain, apiKey } = Config; const userKey = getDestinationExternalID(message, 'userKey'); if (isDefinedAndNotNullAndNotEmpty(userKey)) { - return getUserByUserKey(apiKey, userKey, appSubdomain); + return getUserByUserKey(apiKey, userKey, appSubdomain, metadata); } const integrationsObj = getIntegrationsObj(message, 'user'); @@ -504,11 +511,11 @@ const retrieveUserFromLookup = async (message, destination) => { const lookupFieldValue = getFieldValueFromMessage(message, lookupField); if (lookupField === 'email') { - return getUserByEmail(apiKey, lookupFieldValue, appSubdomain); + return getUserByEmail(apiKey, lookupFieldValue, appSubdomain, metadata); } if (lookupField === 'phone') { - return getUserByPhoneNumber(apiKey, lookupFieldValue, appSubdomain); + return getUserByPhoneNumber(apiKey, lookupFieldValue, appSubdomain, metadata); } throw new InstrumentationError( @@ -517,11 +524,11 @@ const retrieveUserFromLookup = async (message, destination) => { } else { const userId = getValueFromMessage(message, 'userId'); if (userId) { - return getUserByCustomId(message, destination); + return getUserByCustomId(message, destination, metadata); } const email = getFieldValueFromMessage(message, 'email'); if (isDefinedAndNotNullAndNotEmpty(email)) { - return getUserByEmail(apiKey, email, appSubdomain); + return getUserByEmail(apiKey, email, appSubdomain, metadata); } throw new InstrumentationError('Default lookup field : email value is empty'); @@ -535,7 +542,7 @@ const retrieveUserFromLookup = async (message, destination) => { * @param {*} id * @returns */ -const createOrUpdateUserPayloadBuilder = (message, destination, id = null) => { +const createOrUpdateUserPayloadBuilder = ({ message, destination }, id = null) => { const { appSubdomain } = destination.Config; const commonUserPropertiesPayload = constructPayload( message, diff --git a/src/v0/destinations/wootric/transform.js b/src/v0/destinations/wootric/transform.js index f8b4274af7..940d6e9e5d 100644 --- a/src/v0/destinations/wootric/transform.js +++ b/src/v0/destinations/wootric/transform.js @@ -37,17 +37,17 @@ const responseBuilder = async (payload, endpoint, method, accessToken) => { throw new TransformationError('Something went wrong while constructing the payload'); }; -const identifyResponseBuilder = async (message, destination) => { +const identifyResponseBuilder = async ({ message, destination, metadata }) => { let payload; let endpoint; let method; let builder; - const accessToken = await getAccessToken(destination); + const accessToken = await getAccessToken(destination, metadata); const rawEndUserId = getDestinationExternalID(message, 'wootricEndUserId'); const userId = getFieldValueFromMessage(message, 'userIdOnly'); - const userDetails = await retrieveUserDetails(rawEndUserId, userId, accessToken); + const userDetails = await retrieveUserDetails(rawEndUserId, userId, accessToken, metadata); const wootricEndUserId = userDetails?.id; // If user already exist we will update it else creates a new user @@ -132,7 +132,7 @@ const trackResponseBuilder = async (message, destination) => { return responseBuilder(payload, endpoint, method, accessToken); }; -const processEvent = async (message, destination) => { +const processEvent = async ({ message, destination, metadata }) => { if (!message.type) { throw new InstrumentationError('Event type is required'); } @@ -140,7 +140,7 @@ const processEvent = async (message, destination) => { let response; switch (messageType) { case EventType.IDENTIFY: - response = await identifyResponseBuilder(message, destination); + response = await identifyResponseBuilder({ message, destination, metadata }); break; case EventType.TRACK: response = await trackResponseBuilder(message, destination); @@ -151,7 +151,7 @@ const processEvent = async (message, destination) => { return response; }; -const process = async (event) => processEvent(event.message, event.destination); +const process = async (event) => processEvent(event); const processRouterDest = async (inputs, reqMetadata) => { const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); diff --git a/src/v0/destinations/wootric/util.js b/src/v0/destinations/wootric/util.js index c2505c635b..398fe94c7d 100644 --- a/src/v0/destinations/wootric/util.js +++ b/src/v0/destinations/wootric/util.js @@ -20,7 +20,7 @@ const ACCESS_TOKEN_CACHE = new Cache(ACCESS_TOKEN_CACHE_TTL_SECONDS); * @param {*} destination * @returns */ -const getAccessToken = async (destination) => { +const getAccessToken = async (destination, metadata) => { const { username, password, accountToken } = destination.Config; const accessTokenKey = destination.ID; @@ -49,6 +49,7 @@ const getAccessToken = async (destination) => { endpointPath: `/oauth/token`, requestMethod: 'POST', module: 'router', + metadata, }); const processedAuthResponse = processAxiosResponse(wootricAuthResponse); // If the request fails, throwing error. @@ -79,7 +80,7 @@ const getAccessToken = async (destination) => { * @returns */ -const retrieveUserDetails = async (endUserId, externalId, accessToken) => { +const retrieveUserDetails = async (endUserId, externalId, accessToken, metadata) => { let endpoint; if (isDefinedAndNotNullAndNotEmpty(endUserId)) { endpoint = `${BASE_ENDPOINT}/${VERSION}/end_users/${endUserId}`; @@ -104,6 +105,7 @@ const retrieveUserDetails = async (endUserId, externalId, accessToken) => { endpointPath: `/v1/end_users/`, requestMethod: 'GET', module: 'router', + metadata, }); const processedUserResponse = processAxiosResponse(userResponse); diff --git a/src/v0/destinations/yahoo_dsp/transform.js b/src/v0/destinations/yahoo_dsp/transform.js index 4cd1eee73d..f11f1629a8 100644 --- a/src/v0/destinations/yahoo_dsp/transform.js +++ b/src/v0/destinations/yahoo_dsp/transform.js @@ -21,7 +21,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); * @param {*} destination * @returns */ -const responseBuilder = async (message, destination) => { +const responseBuilder = async (message, destination, metadata) => { let dspListPayload = {}; const { Config } = destination; const { listData } = message.properties; @@ -72,7 +72,7 @@ const responseBuilder = async (message, destination) => { response.endpoint = `${BASE_ENDPOINT}/traffic/audiences/${ENDPOINTS[audienceType]}/${audienceId}`; response.body.JSON = removeUndefinedAndNullValues(dspListPayload); response.method = defaultPutRequestConfig.requestMethod; - const accessToken = await getAccessToken(destination); + const accessToken = await getAccessToken(destination, metadata); response.headers = { 'X-Auth-Token': accessToken, 'X-Auth-Method': 'OAuth2', @@ -81,7 +81,7 @@ const responseBuilder = async (message, destination) => { return response; }; -const processEvent = async (message, destination) => { +const processEvent = async ({ message, destination, metadata }) => { let response; if (!message.type) { throw new InstrumentationError('Event type is required'); @@ -93,14 +93,14 @@ const processEvent = async (message, destination) => { throw new InstrumentationError('listData is not present inside properties. Aborting message'); } if (message.type.toLowerCase() === 'audiencelist') { - response = await responseBuilder(message, destination); + response = await responseBuilder(message, destination, metadata); } else { throw new InstrumentationError(`Event type ${message.type} is not supported`, 400); } return response; }; -const process = async (event) => processEvent(event.message, event.destination); +const process = async (event) => processEvent(event); const processRouterDest = async (inputs, reqMetadata) => { const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); diff --git a/src/v0/destinations/yahoo_dsp/util.js b/src/v0/destinations/yahoo_dsp/util.js index 54002a3bce..ba19ac7725 100644 --- a/src/v0/destinations/yahoo_dsp/util.js +++ b/src/v0/destinations/yahoo_dsp/util.js @@ -95,7 +95,7 @@ const createPayload = (audienceList, Config) => { * @param {*} destination * @returns */ -const getAccessToken = async (destination) => { +const getAccessToken = async (destination, metadata) => { const { clientId, clientSecret } = destination.Config; const accessTokenKey = destination.ID; @@ -140,6 +140,7 @@ const getAccessToken = async (destination) => { endpointPath: '/identity/oauth2/access_token', requestMethod: 'POST', module: 'router', + metadata, }); // If the request fails, throwing error. if (dspAuthorisationData.success === false) { diff --git a/src/v0/destinations/zendesk/transform.js b/src/v0/destinations/zendesk/transform.js index cadb1d3964..792e8df350 100644 --- a/src/v0/destinations/zendesk/transform.js +++ b/src/v0/destinations/zendesk/transform.js @@ -95,7 +95,13 @@ const responseBuilderToUpdatePrimaryAccount = ( * @param {*} headers -> Authorizations for API's call * @returns it return payloadbuilder for updating email */ -const payloadBuilderforUpdatingEmail = async (userId, headers, userEmail, baseEndpoint) => { +const payloadBuilderforUpdatingEmail = async ( + userId, + headers, + userEmail, + baseEndpoint, + metadata, +) => { // url for list all identities of user const url = `${baseEndpoint}users/${userId}/identities`; const config = { headers }; @@ -106,6 +112,7 @@ const payloadBuilderforUpdatingEmail = async (userId, headers, userEmail, baseEn endpointPath: 'users/userId/identities', requestMethod: 'POST', module: 'router', + metadata, }); if (res?.response?.data?.count > 0) { const { identities } = res.response.data; @@ -131,7 +138,7 @@ const payloadBuilderforUpdatingEmail = async (userId, headers, userEmail, baseEn return {}; }; -async function createUserFields(url, config, newFields, fieldJson) { +async function createUserFields(url, config, newFields, fieldJson, metadata) { let fieldData; // removing trailing 's' from fieldJson const fieldJsonSliced = fieldJson.slice(0, -1); @@ -154,6 +161,7 @@ async function createUserFields(url, config, newFields, fieldJson) { endpointPath: '/users/userId/identities', requestMethod: 'POST', module: 'router', + metadata, }); if (response.status !== 201) { logger.debug(`${NAME}:: Failed to create User Field : `, field); @@ -173,6 +181,7 @@ async function checkAndCreateUserFields( fieldJson, headers, baseEndpoint, + metadata, ) { let newFields = []; @@ -185,6 +194,7 @@ async function checkAndCreateUserFields( feature: 'transformation', requestMethod: 'POST', module: 'router', + metadata, }); const fields = get(response.data, fieldJson); if (response.data && fields) { @@ -199,7 +209,7 @@ async function checkAndCreateUserFields( ); if (newFields.length > 0) { - await createUserFields(url, config, newFields, fieldJson); + await createUserFields(url, config, newFields, fieldJson, metadata); } } } catch (error) { @@ -249,7 +259,7 @@ function getIdentifyPayload(message, category, destinationConfig, type) { * @param {*} headers headers for authorizations * @returns */ -const getUserIdByExternalId = async (message, headers, baseEndpoint) => { +const getUserIdByExternalId = async (message, headers, baseEndpoint, metadata) => { const externalId = getFieldValueFromMessage(message, 'userIdOnly'); if (!externalId) { logger.debug(`${NAME}:: externalId is required for getting zenuserId`); @@ -265,6 +275,7 @@ const getUserIdByExternalId = async (message, headers, baseEndpoint) => { endpointPath, requestMethod: 'GET', module: 'router', + metadata, }); if (resp?.response?.data?.count > 0) { @@ -278,7 +289,7 @@ const getUserIdByExternalId = async (message, headers, baseEndpoint) => { return undefined; }; -async function getUserId(message, headers, baseEndpoint, type) { +async function getUserId(message, headers, baseEndpoint, type, metadata) { const traits = type === 'group' ? get(message, CONTEXT_TRAITS_KEY_PATH) @@ -298,6 +309,7 @@ async function getUserId(message, headers, baseEndpoint, type) { endpointPath, requestMethod: 'GET', module: 'router', + metadata, }); if (!resp || !resp.data || resp.data.count === 0) { logger.debug(`${NAME}:: User not found`); @@ -315,7 +327,7 @@ async function getUserId(message, headers, baseEndpoint, type) { } } -async function isUserAlreadyAssociated(userId, orgId, headers, baseEndpoint) { +async function isUserAlreadyAssociated(userId, orgId, headers, baseEndpoint, metadata) { const url = `${baseEndpoint}/users/${userId}/organization_memberships.json`; const config = { headers }; try { @@ -325,6 +337,7 @@ async function isUserAlreadyAssociated(userId, orgId, headers, baseEndpoint) { endpointPath: '/users/userId/organization_memberships.json', requestMethod: 'GET', module: 'router', + metadata, }); if (response?.data?.organization_memberships?.[0]?.organization_id === orgId) { return true; @@ -336,7 +349,7 @@ async function isUserAlreadyAssociated(userId, orgId, headers, baseEndpoint) { return false; } -async function createUser(message, headers, destinationConfig, baseEndpoint, type) { +async function createUser(message, headers, destinationConfig, baseEndpoint, type, metadata) { const traits = type === 'group' ? get(message, CONTEXT_TRAITS_KEY_PATH) @@ -360,6 +373,7 @@ async function createUser(message, headers, destinationConfig, baseEndpoint, typ endpointPath: '/users/create_or_update.json', requestMethod: 'POST', module: 'router', + metadata, }); if (!resp.data || !resp.data.user || !resp.data.user.id) { @@ -377,9 +391,16 @@ async function createUser(message, headers, destinationConfig, baseEndpoint, typ } } -async function getUserMembershipPayload(message, headers, orgId, destinationConfig, baseEndpoint) { +async function getUserMembershipPayload( + message, + headers, + orgId, + destinationConfig, + baseEndpoint, + metadata, +) { // let zendeskUserID = await getUserId(message.userId, headers); - let zendeskUserID = await getUserId(message, headers, baseEndpoint, 'group'); + let zendeskUserID = await getUserId(message, headers, baseEndpoint, 'group', metadata); const traits = get(message, CONTEXT_TRAITS_KEY_PATH); if (!zendeskUserID) { if (traits && traits.name && traits.email) { @@ -389,6 +410,7 @@ async function getUserMembershipPayload(message, headers, orgId, destinationConf destinationConfig, baseEndpoint, 'group', + metadata, ); zendeskUserID = zendeskUserId; } else { @@ -405,7 +427,14 @@ async function getUserMembershipPayload(message, headers, orgId, destinationConf return payload; } -async function createOrganization(message, category, headers, destinationConfig, baseEndpoint) { +async function createOrganization( + message, + category, + headers, + destinationConfig, + baseEndpoint, + metadata, +) { if (!isDefinedAndNotNull(message.traits)) { throw new InstrumentationError('Organisation Traits are missing. Aborting.'); } @@ -415,6 +444,7 @@ async function createOrganization(message, category, headers, destinationConfig, category.organizationFieldsJson, headers, baseEndpoint, + metadata, ); const mappingJson = mappingConfig[category.name]; const payload = constructPayload(message, mappingJson); @@ -447,6 +477,7 @@ async function createOrganization(message, category, headers, destinationConfig, endpointPath: '/organizations/create_or_update.json', requestMethod: 'POST', module: 'router', + metadata, }); if (!resp.data || !resp.data.organization) { @@ -468,7 +499,7 @@ function validateUserId(message) { } } -async function processIdentify(message, destinationConfig, headers, baseEndpoint) { +async function processIdentify(message, destinationConfig, headers, baseEndpoint, metadata) { validateUserId(message); const category = ConfigCategory.IDENTIFY; const traits = getFieldValueFromMessage(message, 'traits'); @@ -480,6 +511,7 @@ async function processIdentify(message, destinationConfig, headers, baseEndpoint category.userFieldsJson, headers, baseEndpoint, + metadata, ); const payload = getIdentifyPayload(message, category, destinationConfig, 'identify'); @@ -487,7 +519,12 @@ async function processIdentify(message, destinationConfig, headers, baseEndpoint const returnList = []; if (destinationConfig.searchByExternalId) { - const userIdByExternalId = await getUserIdByExternalId(message, headers, baseEndpoint); + const userIdByExternalId = await getUserIdByExternalId( + message, + headers, + baseEndpoint, + metadata, + ); const userEmail = traits?.email; if (userIdByExternalId && userEmail) { const payloadForUpdatingEmail = await payloadBuilderforUpdatingEmail( @@ -495,6 +532,7 @@ async function processIdentify(message, destinationConfig, headers, baseEndpoint headers, userEmail, baseEndpoint, + metadata, ); if (!isEmptyObject(payloadForUpdatingEmail)) returnList.push(payloadForUpdatingEmail); } @@ -507,7 +545,7 @@ async function processIdentify(message, destinationConfig, headers, baseEndpoint traits.company.id ) { const orgId = traits.company.id; - const userId = await getUserId(message, headers, baseEndpoint); + const userId = await getUserId(message, headers, baseEndpoint, metadata); if (userId) { const membershipUrl = `${baseEndpoint}users/${userId}/organization_memberships.json`; try { @@ -518,6 +556,7 @@ async function processIdentify(message, destinationConfig, headers, baseEndpoint endpointPath: '/users/userId/organization_memberships.json', requestMethod: 'GET', module: 'router', + metadata, }); if ( response.data && @@ -547,7 +586,7 @@ async function processIdentify(message, destinationConfig, headers, baseEndpoint return returnList; } -async function processTrack(message, destinationConfig, headers, baseEndpoint) { +async function processTrack(message, destinationConfig, headers, baseEndpoint, metadata) { validateUserId(message); const traits = getFieldValueFromMessage(message, 'traits'); let userEmail; @@ -568,6 +607,7 @@ async function processTrack(message, destinationConfig, headers, baseEndpoint) { endpointPath, requestMethod: 'GET', module: 'router', + metadata, }); if (!get(userResponse, 'data.users.0.id') || userResponse.data.count === 0) { const { zendeskUserId, email } = await createUser( @@ -575,6 +615,7 @@ async function processTrack(message, destinationConfig, headers, baseEndpoint) { headers, destinationConfig, baseEndpoint, + metadata, ); if (!zendeskUserId) { throw new NetworkInstrumentationError('User not found'); @@ -618,13 +659,20 @@ async function processTrack(message, destinationConfig, headers, baseEndpoint) { return response; } -async function processGroup(message, destinationConfig, headers, baseEndpoint) { +async function processGroup(message, destinationConfig, headers, baseEndpoint, metadata) { const category = ConfigCategory.GROUP; let payload; let url; if (destinationConfig.sendGroupCallsWithoutUserId && !message.userId) { - payload = await createOrganization(message, category, headers, destinationConfig, baseEndpoint); + payload = await createOrganization( + message, + category, + headers, + destinationConfig, + baseEndpoint, + metadata, + ); url = baseEndpoint + category.createEndpoint; } else { validateUserId(message); @@ -634,6 +682,7 @@ async function processGroup(message, destinationConfig, headers, baseEndpoint) { headers, destinationConfig, baseEndpoint, + metadata, ); if (!orgId) { throw new NetworkInstrumentationError( @@ -648,11 +697,12 @@ async function processGroup(message, destinationConfig, headers, baseEndpoint) { orgId, destinationConfig, baseEndpoint, + metadata, ); url = baseEndpoint + category.userMembershipEndpoint; const userId = payload.organization_membership.user_id; - if (await isUserAlreadyAssociated(userId, orgId, headers, baseEndpoint)) { + if (await isUserAlreadyAssociated(userId, orgId, headers, baseEndpoint, metadata)) { throw new InstrumentationError('User is already associated with organization'); } } @@ -678,15 +728,15 @@ async function processSingleMessage(event) { 'Content-Type': JSON_MIME_TYPE, }; - const { message } = event; + const { message, metadata } = event; const evType = getEventType(message); switch (evType) { case EventType.IDENTIFY: - return processIdentify(message, destinationConfig, headers, baseEndpoint); + return processIdentify(message, destinationConfig, headers, baseEndpoint, metadata); case EventType.GROUP: - return processGroup(message, destinationConfig, headers, baseEndpoint); + return processGroup(message, destinationConfig, headers, baseEndpoint, metadata); case EventType.TRACK: - return processTrack(message, destinationConfig, headers, baseEndpoint); + return processTrack(message, destinationConfig, headers, baseEndpoint, metadata); default: throw new InstrumentationError(`Event type ${evType} is not supported`); } diff --git a/src/v0/util/index.js b/src/v0/util/index.js index dc06bcc43a..fa6cc34b70 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -2270,6 +2270,22 @@ const validateEventAndLowerCaseConversion = (event, isMandatory, convertToLowerC const applyCustomMappings = (message, mappings) => JsonTemplateEngine.createAsSync(mappings, { defaultPathType: PathType.JSON }).evaluate(message); +/** + * Gets url path omitting the hostname & protocol + * + * **Note**: + * - This should only be used when there are no dynamic paths in URL + * @param {*} inputUrl + * @returns + */ +const getRelativePathFromURL = (inputUrl) => { + if (isValidUrl(inputUrl)) { + const url = new URL(inputUrl); + return url.pathname; + } + return inputUrl; +}; + // ======================================================================== // EXPORTS // ======================================================================== @@ -2390,5 +2406,6 @@ module.exports = { removeDuplicateMetadata, combineBatchRequestsWithSameJobIds, validateEventAndLowerCaseConversion, + getRelativePathFromURL, removeEmptyKey, }; diff --git a/src/v0/util/index.test.js b/src/v0/util/index.test.js index c34d513325..31ea490b25 100644 --- a/src/v0/util/index.test.js +++ b/src/v0/util/index.test.js @@ -690,3 +690,24 @@ describe('extractCustomFields', () => { }); }); }); + +describe('get relative path from url', () => { + test('valid url', () => { + expect(utilities.getRelativePathFromURL('https://google.com/a/b/c')).toEqual('/a/b/c'); + }); + test('valid url with query parameters', () => { + expect(utilities.getRelativePathFromURL('https://google.com/a/b/c?q=1&n=2')).toEqual('/a/b/c'); + }); + test('normal string', () => { + expect(utilities.getRelativePathFromURL('s=1&n=2')).toEqual('s=1&n=2'); + }); + test('undefined', () => { + expect(utilities.getRelativePathFromURL(undefined)).toEqual(undefined); + }); + test('number', () => { + expect(utilities.getRelativePathFromURL(1)).toEqual(1); + }); + test('null', () => { + expect(utilities.getRelativePathFromURL(null)).toEqual(null); + }); +}); From b7860a5b2b87fb61aaff8c68a904ac996d63efd3 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Wed, 24 Jul 2024 12:36:46 +0530 Subject: [PATCH 17/28] fix: add validation for cordial destination (#3599) * fix: add validation for cordial destination * chore: fix linting --- .../v2/destinations/cordial/procWorkflow.yaml | 15 +++++-- .../cordial/processor/validation.ts | 43 ++++++++++++++++++- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/cdk/v2/destinations/cordial/procWorkflow.yaml b/src/cdk/v2/destinations/cordial/procWorkflow.yaml index 2c765afeb5..641cf28b06 100644 --- a/src/cdk/v2/destinations/cordial/procWorkflow.yaml +++ b/src/cdk/v2/destinations/cordial/procWorkflow.yaml @@ -35,13 +35,16 @@ steps: template: | $.context.messageType = .message.type.toLowerCase(); - - name: validateInput + - name: validateConfig + template: | + $.assertConfig(.destination.Config.apiKey, "API Key is not present. Aborting"); + $.assertConfig(.destination.Config.apiBaseUrl, "API Base URl is not present. Aborting"); + + - name: validateMessageType template: | let messageType = $.context.messageType; $.assert(messageType, "message Type is not present. Aborting"); $.assert(messageType in {{$.EventType.([.TRACK, .IDENTIFY])}}, "message type " + messageType + " is not supported"); - $.assertConfig(.destination.Config.apiKey, "API Key is not present. Aborting"); - $.assertConfig(.destination.Config.apiBaseUrl, "API Base URl is not present. Aborting"); - name: getContactId template: | @@ -49,7 +52,11 @@ steps: - name: getContactEmail template: | - .message.().({{{{$.getGenericPaths("email")}}}};); + .message.().({{{{$.getGenericPaths("emailOnly")}}}}); + + - name: validateEventPayload + template: | + $.assert($.outputs.getContactId || $.outputs.getContactEmail, "Either one of cordial contact id or email is required. Aborting"); - name: buildIdentifyPayload condition: $.context.messageType in [{{$.EventType.IDENTIFY}}] diff --git a/test/integrations/destinations/cordial/processor/validation.ts b/test/integrations/destinations/cordial/processor/validation.ts index a764b79443..61f2e1cbed 100644 --- a/test/integrations/destinations/cordial/processor/validation.ts +++ b/test/integrations/destinations/cordial/processor/validation.ts @@ -3,6 +3,47 @@ import { generateMetadata } from '../../../testUtils'; import { destType, destination, processorInstrumentationErrorStatTags } from '../common'; export const validation: ProcessorTestData[] = [ + { + id: 'cordial-validation-test-1', + name: destType, + description: 'All of the required fields — cordial contact id, email — are missing.', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Either one of cordial contact id or email is required. Aborting: Workflow: procWorkflow, Step: validateEventPayload, ChildStep: undefined, OriginalError: Either one of cordial contact id or email is required. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, { id: 'cordial-validation-test-2', name: destType, @@ -38,7 +79,7 @@ export const validation: ProcessorTestData[] = [ body: [ { error: - 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + 'message type group is not supported: Workflow: procWorkflow, Step: validateMessageType, ChildStep: undefined, OriginalError: message type group is not supported', metadata: generateMetadata(1), statTags: processorInstrumentationErrorStatTags, statusCode: 400, From 895b327340a4eb14de884f386812135c9a8527c2 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Wed, 24 Jul 2024 07:10:33 +0000 Subject: [PATCH 18/28] chore(release): 1.72.3 --- CHANGELOG.md | 8 ++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39548ce37c..2223b8aa2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.72.3](https://github.com/rudderlabs/rudder-transformer/compare/v1.72.2...v1.72.3) (2024-07-24) + + +### Bug Fixes + +* add validation for cordial destination ([#3599](https://github.com/rudderlabs/rudder-transformer/issues/3599)) ([b7860a5](https://github.com/rudderlabs/rudder-transformer/commit/b7860a5b2b87fb61aaff8c68a904ac996d63efd3)) +* update getConversionActionId function for gaoc ([#3594](https://github.com/rudderlabs/rudder-transformer/issues/3594)) ([68367f5](https://github.com/rudderlabs/rudder-transformer/commit/68367f5227c96f2700a773018b991b1e87a0774d)) + ### [1.72.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.72.1...v1.72.2) (2024-07-23) ### [1.72.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.72.0...v1.72.1) (2024-07-23) diff --git a/package-lock.json b/package-lock.json index c4619cc586..7987a1a1ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.72.2", + "version": "1.72.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.72.2", + "version": "1.72.3", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 94cf607c50..f47b1eb124 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.72.2", + "version": "1.72.3", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 3ddf9e208702f27cbba80b032c3ed681a70c518f Mon Sep 17 00:00:00 2001 From: kanishkkatara <58228944+kanishkkatara@users.noreply.github.com> Date: Wed, 24 Jul 2024 13:59:39 +0530 Subject: [PATCH 19/28] fix: codeowners file (#3601) --- CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 274166f613..702a461d92 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -7,5 +7,5 @@ src/features.json @rudderlabs/integrations constants/ @rudderlabs/integrations warehouse/ @rudderlabs/warehouse src/util/ @rudderlabs/integrations @rudderlabs/data-management -*/trackingPlan.ts @rudderlabs/data-management -*/userTransform.ts @rudderlabs/data-management +**/trackingPlan.ts @rudderlabs/data-management +**/userTransform.ts @rudderlabs/data-management From ef1c4f9401b6bec05cca170e644355db96a5a32c Mon Sep 17 00:00:00 2001 From: kanishkkatara <58228944+kanishkkatara@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:07:04 +0530 Subject: [PATCH 20/28] chore: capturing user_transform_test_errors stat (#3543) * chore: capturing user_transform_test_errors stat * fix: error code * fix: lint error * chore: changed error stat to total request stat --- src/services/userTransform.ts | 8 ++++++++ src/util/prometheus.js | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/src/services/userTransform.ts b/src/services/userTransform.ts index 9ad92e8ca3..2afad88c56 100644 --- a/src/services/userTransform.ts +++ b/src/services/userTransform.ts @@ -196,6 +196,7 @@ export class UserTransformService { public static async testTransformRoutine(events, trRevCode, libraryVersionIDs, credentials) { const response: FixMe = {}; + let errorCode: number | undefined; try { if (!trRevCode || !trRevCode.code || !trRevCode.codeVersion) { throw new Error('Invalid Request. Missing parameters in transformation code block'); @@ -231,6 +232,13 @@ export class UserTransformService { response.body = { error: extractStackTraceUptoLastSubstringMatch(error.stack, SUPPORTED_FUNC_NAMES), }; + errorCode = error.statusCode; + } finally { + const metaTags = getTransformationMetadata(events[0]?.metadata); + stats.counter('user_transform_test_count_total', events.length, { + status: errorCode || response.status, + ...metaTags, + }); } return response; } diff --git a/src/util/prometheus.js b/src/util/prometheus.js index 49f1fdcd8b..cddeb80f31 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -841,6 +841,12 @@ class Prometheus { 'k8_namespace', ], }, + { + name: 'user_transform_test_count_total', + help: 'user_transform_test_count_total', + type: 'counter', + labelNames: ['workspaceId', 'transformationId', 'status'], + }, { name: 'user_transform_requests', help: 'user_transform_requests', From d74178f81911f029d897eed6d33939a0115829ae Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Wed, 24 Jul 2024 14:48:42 +0530 Subject: [PATCH 21/28] fix: update response handler for array type response --- .../networkHandler.js | 2 +- src/v0/util/googleUtils/index.js | 11 ++--- .../dataDelivery/oauth.ts | 4 +- .../network.ts | 44 ++++++++++--------- 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/v0/destinations/google_adwords_offline_conversions/networkHandler.js b/src/v0/destinations/google_adwords_offline_conversions/networkHandler.js index f260abeca5..df69a95299 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/networkHandler.js +++ b/src/v0/destinations/google_adwords_offline_conversions/networkHandler.js @@ -43,7 +43,7 @@ const createJob = async ({ endpoint, headers, payload, metadata }) => { const { response, status } = createJobResponse; if (!isHttpStatusSuccess(status)) { throw new AbortedError( - `[Google Ads Offline Conversions]:: ${response?.error?.message} during google_ads_offline_store_conversions Job Creation`, + `[Google Ads Offline Conversions]:: ${response?.error?.message || response?.[0]?.error?.message} during google_ads_offline_store_conversions Job Creation`, status, response, getAuthErrCategory(createJobResponse), diff --git a/src/v0/util/googleUtils/index.js b/src/v0/util/googleUtils/index.js index 183f327fa4..3bb02d1b83 100644 --- a/src/v0/util/googleUtils/index.js +++ b/src/v0/util/googleUtils/index.js @@ -114,11 +114,12 @@ const finaliseAnalyticsConsents = (consentConfigMap, eventLevelConsent = {}) => const getAuthErrCategory = ({ response, status }) => { if (status === 401) { - const authenticationError = get( - response, - 'error.details.0.errors.0.errorCode.authenticationError', - ); - if (authenticationError === 'TWO_STEP_VERIFICATION_NOT_ENROLLED') { + let respArr = response; + if (!Array.isArray(response)) { + respArr = [response]; + } + const authenticationError = respArr.map((resp) => get(resp, 'error.details.0.errors.0.errorCode.authenticationError')); + if (authenticationError.includes('TWO_STEP_VERIFICATION_NOT_ENROLLED')) { // https://developers.google.com/google-ads/api/docs/oauth/2sv return AUTH_STATUS_INACTIVE; } diff --git a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts index b8e4c78e1d..4437ebb912 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts @@ -368,11 +368,11 @@ export const v1oauthScenarios = [ output: { authErrorCategory: 'AUTH_STATUS_INACTIVE', message: - '[Google Ads Offline Conversions]:: {"error":{"code":401,"details":[{"@type":"type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure","errors":[{"errorCode":{"authenticationError":"TWO_STEP_VERIFICATION_NOT_ENROLLED"},"message":"An account administrator changed this account\'s authentication settings. To access this Google Ads account, enable 2-Step Verification in your Google account at https://www.google.com/landing/2step."}],"requestId":"wy4ZYbsjWcgh6uC2Ruc_Zg"}],"message":"Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.","status":"UNAUTHENTICATED"}} during google_ads_offline_conversions response transformation', + '[Google Ads Offline Conversions]:: [{"error":{"code":401,"details":[{"@type":"type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure","errors":[{"errorCode":{"authenticationError":"TWO_STEP_VERIFICATION_NOT_ENROLLED"},"message":"An account administrator changed this account\'s authentication settings. To access this Google Ads account, enable 2-Step Verification in your Google account at https://www.google.com/landing/2step."}],"requestId":"wy4ZYbsjWcgh6uC2Ruc_Zg"}],"message":"Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.","status":"UNAUTHENTICATED"}}] during google_ads_offline_conversions response transformation', response: [ { error: - '[Google Ads Offline Conversions]:: {"error":{"code":401,"details":[{"@type":"type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure","errors":[{"errorCode":{"authenticationError":"TWO_STEP_VERIFICATION_NOT_ENROLLED"},"message":"An account administrator changed this account\'s authentication settings. To access this Google Ads account, enable 2-Step Verification in your Google account at https://www.google.com/landing/2step."}],"requestId":"wy4ZYbsjWcgh6uC2Ruc_Zg"}],"message":"Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.","status":"UNAUTHENTICATED"}} during google_ads_offline_conversions response transformation', + '[Google Ads Offline Conversions]:: [{"error":{"code":401,"details":[{"@type":"type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure","errors":[{"errorCode":{"authenticationError":"TWO_STEP_VERIFICATION_NOT_ENROLLED"},"message":"An account administrator changed this account\'s authentication settings. To access this Google Ads account, enable 2-Step Verification in your Google account at https://www.google.com/landing/2step."}],"requestId":"wy4ZYbsjWcgh6uC2Ruc_Zg"}],"message":"Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.","status":"UNAUTHENTICATED"}}] during google_ads_offline_conversions response transformation', metadata: { attemptNum: 1, destinationId: 'default-destinationId', diff --git a/test/integrations/destinations/google_adwords_offline_conversions/network.ts b/test/integrations/destinations/google_adwords_offline_conversions/network.ts index f1147162d2..0ab6bef1db 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/network.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/network.ts @@ -1,28 +1,30 @@ const commonResponse = { status: 401, - data: { - error: { - code: 401, - details: [ - { - '@type': 'type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure', - errors: [ - { - errorCode: { - authenticationError: 'TWO_STEP_VERIFICATION_NOT_ENROLLED', + data: [ + { + error: { + code: 401, + details: [ + { + '@type': 'type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure', + errors: [ + { + errorCode: { + authenticationError: 'TWO_STEP_VERIFICATION_NOT_ENROLLED', + }, + message: + "An account administrator changed this account's authentication settings. To access this Google Ads account, enable 2-Step Verification in your Google account at https://www.google.com/landing/2step.", }, - message: - "An account administrator changed this account's authentication settings. To access this Google Ads account, enable 2-Step Verification in your Google account at https://www.google.com/landing/2step.", - }, - ], - requestId: 'wy4ZYbsjWcgh6uC2Ruc_Zg', - }, - ], - message: - 'Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.', - status: 'UNAUTHENTICATED', + ], + requestId: 'wy4ZYbsjWcgh6uC2Ruc_Zg', + }, + ], + message: + 'Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.', + status: 'UNAUTHENTICATED', + }, }, - }, + ], }; export const networkCallsData = [ From 1f9b83ff6cc24a604c2373f10a1fedb5f879ea3c Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Wed, 24 Jul 2024 14:53:41 +0530 Subject: [PATCH 22/28] fix: lint error --- src/v0/util/googleUtils/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/v0/util/googleUtils/index.js b/src/v0/util/googleUtils/index.js index 3bb02d1b83..406afa1a49 100644 --- a/src/v0/util/googleUtils/index.js +++ b/src/v0/util/googleUtils/index.js @@ -118,7 +118,9 @@ const getAuthErrCategory = ({ response, status }) => { if (!Array.isArray(response)) { respArr = [response]; } - const authenticationError = respArr.map((resp) => get(resp, 'error.details.0.errors.0.errorCode.authenticationError')); + const authenticationError = respArr.map((resp) => + get(resp, 'error.details.0.errors.0.errorCode.authenticationError'), + ); if (authenticationError.includes('TWO_STEP_VERIFICATION_NOT_ENROLLED')) { // https://developers.google.com/google-ads/api/docs/oauth/2sv return AUTH_STATUS_INACTIVE; From ebf9e3fa785a2372e680cec65a9a27187d7f0415 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Wed, 24 Jul 2024 19:08:22 +0530 Subject: [PATCH 23/28] fix: queryID aloglia discrepancy (#3597) --- .../v2/destinations/algolia/procWorkflow.yaml | 2 +- .../destinations/algolia/processor/data.ts | 141 ++++++++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/src/cdk/v2/destinations/algolia/procWorkflow.yaml b/src/cdk/v2/destinations/algolia/procWorkflow.yaml index 402b48dabd..44741b5f60 100644 --- a/src/cdk/v2/destinations/algolia/procWorkflow.yaml +++ b/src/cdk/v2/destinations/algolia/procWorkflow.yaml @@ -52,7 +52,7 @@ steps: template: | const products = ^.message.properties.products products.($.removeUndefinedAndNullValues({ - "queryID" : $.isDefinedAndNotNull(.queryID) ? String(.queryID) : null, + "queryID" : $.isDefinedAndNotNull(.queryID || .queryId) ? String(.queryID || .queryId) : null, "price": $.isDefinedAndNotNull(.price) && $.isDefinedAndNotNull(^.message.properties.currency) ? String(.price) : null, "quantity": $.isDefinedAndNotNull(.quantity)? Number(.quantity) : null, "discount": $.isDefinedAndNotNull(.discount) ? String(.discount) : null diff --git a/test/integrations/destinations/algolia/processor/data.ts b/test/integrations/destinations/algolia/processor/data.ts index d239c8de70..b37b6e4246 100644 --- a/test/integrations/destinations/algolia/processor/data.ts +++ b/test/integrations/destinations/algolia/processor/data.ts @@ -2429,4 +2429,145 @@ export const data = [ }, }, }, + { + name: 'algolia', + description: 'queryID and queryId inconsistency test', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + traits: { + email: 'testone@gmail.com', + firstName: 'test', + lastName: 'one', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + page: { + path: '/destinations/ometria', + referrer: '', + search: '', + title: '', + url: 'https://docs.rudderstack.com/destinations/ometria', + category: 'destination', + initial_referrer: 'https://docs.rudderstack.com', + initial_referring_domain: 'docs.rudderstack.com', + }, + }, + type: 'track', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + event: 'product list viewed', + userId: 'testuserId1', + properties: { + index: 'products', + eventSubtype: 'purchase', + filters: ['field1:hello', 'val1:val2'], + queryId: '43b15df305339e827f0ac0bdc5ebcaa7', + }, + integrations: { + All: true, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + applicationId: 'O2YARRI15I', + eventTypeSettings: [ + { + from: 'product list viewed', + to: 'conversion', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + body: { + JSON: { + events: [ + { + index: 'products', + queryID: '43b15df305339e827f0ac0bdc5ebcaa7', + filters: ['field1:hello', 'val1:val2'], + userToken: 'testuserId1', + eventName: 'product list viewed', + eventType: 'conversion', + eventSubtype: 'purchase', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://insights.algolia.io/1/events', + headers: { + 'X-Algolia-Application-Id': 'O2YARRI15I', + 'X-Algolia-API-Key': 'dummyApiKey', + }, + params: {}, + files: {}, + userId: '', + }, + statusCode: 200, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + }, ]; From bd8b021248e2e3a615dbbed46804249c0679e88c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 12:11:29 +0530 Subject: [PATCH 24/28] chore(deps): bump docker/build-push-action from 5.1.0 to 6.4.1 (#3583) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5.1.0 to 6.4.1. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v5.1.0...v6.4.1) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-push-docker-image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 0d3494e8d1..57e5fdb3aa 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -51,7 +51,7 @@ jobs: password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} - name: Build Docker Image - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v6.4.1 with: context: . file: ${{ inputs.dockerfile }} @@ -67,7 +67,7 @@ jobs: docker run ${{ inputs.build_tag }} npm run test:ts:ci - name: Build and Push Multi-platform Images - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v6.4.1 with: context: . file: ${{ inputs.dockerfile }} @@ -102,7 +102,7 @@ jobs: password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} - name: Build Docker Image - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v6.4.1 with: context: . file: ${{ inputs.dockerfile }} @@ -118,7 +118,7 @@ jobs: docker run ${{ inputs.build_tag }} npm run test:ts:ci - name: Build and Push Multi-platform Images - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v6.4.1 with: context: . file: ${{ inputs.dockerfile }} From de28dd9ad29ccb4635a3e5d0b131a4b1b33995fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 12:14:58 +0530 Subject: [PATCH 25/28] chore(deps): bump docker/login-action from 2.1.0 to 3.2.0 (#3518) Bumps [docker/login-action](https://github.com/docker/login-action) from 2.1.0 to 3.2.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2.1.0...v3.2.0) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sankeerth --- .github/workflows/build-push-docker-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 57e5fdb3aa..a9844c5c9d 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -45,7 +45,7 @@ jobs: uses: docker/setup-buildx-action@v3.0.0 - name: Login to DockerHub - uses: docker/login-action@v2.1.0 + uses: docker/login-action@v3.2.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} @@ -96,7 +96,7 @@ jobs: uses: docker/setup-buildx-action@v3.0.0 - name: Login to DockerHub - uses: docker/login-action@v2.1.0 + uses: docker/login-action@v3.2.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} @@ -143,7 +143,7 @@ jobs: uses: docker/setup-buildx-action@v3.0.0 - name: Login to DockerHub - uses: docker/login-action@v2.1.0 + uses: docker/login-action@v3.2.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} From 0bc67ad5680ba71fe72fd99833501cf6b72cc69b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 09:21:49 +0200 Subject: [PATCH 26/28] chore(deps): bump actions/setup-go from 5.0.0 to 5.0.2 (#3559) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5.0.0 to 5.0.2. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v5.0.0...v5.0.2) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sai Sankeerth Co-authored-by: Sankeerth --- .github/workflows/ut-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ut-tests.yml b/.github/workflows/ut-tests.yml index 0a2ef8a390..20c20abe24 100644 --- a/.github/workflows/ut-tests.yml +++ b/.github/workflows/ut-tests.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup Go - uses: actions/setup-go@v5.0.0 + uses: actions/setup-go@v5.0.2 with: go-version: 1.17 From ec4db8ddd89b2d7b4fd878966380ad02149ec04d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 09:22:16 +0200 Subject: [PATCH 27/28] chore(deps): bump docker/setup-buildx-action from 3.0.0 to 3.4.0 (#3558) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.0.0 to 3.4.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v3.0.0...v3.4.0) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sankeerth --- .github/workflows/build-push-docker-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index a9844c5c9d..885ecf4fd6 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -42,7 +42,7 @@ jobs: fetch-depth: 1 - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.4.0 - name: Login to DockerHub uses: docker/login-action@v3.2.0 @@ -93,7 +93,7 @@ jobs: fetch-depth: 1 - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.4.0 - name: Login to DockerHub uses: docker/login-action@v3.2.0 @@ -140,7 +140,7 @@ jobs: steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.4.0 - name: Login to DockerHub uses: docker/login-action@v3.2.0 From 99123f14a46a0da95b315ab91282c1c321f90006 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 09:22:49 +0200 Subject: [PATCH 28/28] chore(deps): bump actions/setup-node from 3.7.0 to 4.0.3 (#3557) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3.7.0 to 4.0.3. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v3.7.0...v4.0.3) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sankeerth --- .github/workflows/commitlint.yml | 2 +- .github/workflows/component-test-report.yml | 2 +- .github/workflows/draft-new-release.yml | 2 +- .github/workflows/dt-test-and-report-code-coverage.yml | 2 +- .github/workflows/publish-new-release.yml | 2 +- .github/workflows/ut-tests.yml | 2 +- .github/workflows/verify-server-start.yml | 2 +- .github/workflows/verify.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index a8ff39eee0..1e8889c011 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -12,7 +12,7 @@ jobs: fetch-depth: 0 - name: Setup Node - uses: actions/setup-node@v4.0.1 + uses: actions/setup-node@v4.0.3 with: node-version-file: '.nvmrc' cache: 'npm' diff --git a/.github/workflows/component-test-report.yml b/.github/workflows/component-test-report.yml index 3d457df9ff..936ec53742 100644 --- a/.github/workflows/component-test-report.yml +++ b/.github/workflows/component-test-report.yml @@ -28,7 +28,7 @@ jobs: fetch-depth: 1 - name: Setup Node - uses: actions/setup-node@v4.0.2 + uses: actions/setup-node@v4.0.3 with: node-version-file: '.nvmrc' cache: 'npm' diff --git a/.github/workflows/draft-new-release.yml b/.github/workflows/draft-new-release.yml index 95431fef8e..82174d1d72 100644 --- a/.github/workflows/draft-new-release.yml +++ b/.github/workflows/draft-new-release.yml @@ -16,7 +16,7 @@ jobs: fetch-depth: 0 - name: Setup Node - uses: actions/setup-node@v4.0.2 + uses: actions/setup-node@v4.0.3 with: node-version-file: '.nvmrc' cache: 'npm' diff --git a/.github/workflows/dt-test-and-report-code-coverage.yml b/.github/workflows/dt-test-and-report-code-coverage.yml index 4375b3383e..c3fc6150a6 100644 --- a/.github/workflows/dt-test-and-report-code-coverage.yml +++ b/.github/workflows/dt-test-and-report-code-coverage.yml @@ -20,7 +20,7 @@ jobs: fetch-depth: 1 - name: Setup Node - uses: actions/setup-node@v4.0.2 + uses: actions/setup-node@v4.0.3 with: node-version-file: '.nvmrc' cache: 'npm' diff --git a/.github/workflows/publish-new-release.yml b/.github/workflows/publish-new-release.yml index 15d7b20fd1..9897190575 100644 --- a/.github/workflows/publish-new-release.yml +++ b/.github/workflows/publish-new-release.yml @@ -30,7 +30,7 @@ jobs: fetch-depth: 0 - name: Setup Node - uses: actions/setup-node@v4.0.2 + uses: actions/setup-node@v4.0.3 with: node-version-file: '.nvmrc' cache: 'npm' diff --git a/.github/workflows/ut-tests.yml b/.github/workflows/ut-tests.yml index 20c20abe24..87e7d1fde8 100644 --- a/.github/workflows/ut-tests.yml +++ b/.github/workflows/ut-tests.yml @@ -26,7 +26,7 @@ jobs: fetch-depth: 1 - name: Setup Node - uses: actions/setup-node@v4.0.2 + uses: actions/setup-node@v4.0.3 with: node-version-file: '.nvmrc' cache: 'npm' diff --git a/.github/workflows/verify-server-start.yml b/.github/workflows/verify-server-start.yml index 6ac5b7be05..332434c817 100644 --- a/.github/workflows/verify-server-start.yml +++ b/.github/workflows/verify-server-start.yml @@ -15,7 +15,7 @@ jobs: fetch-depth: 1 - name: Setup Node - uses: actions/setup-node@v4.0.2 + uses: actions/setup-node@v4.0.3 with: node-version-file: '.nvmrc' cache: 'npm' diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4caef8dd91..1ae1caee23 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -16,7 +16,7 @@ jobs: ref: ${{ github.head_ref }} - name: Setup Node - uses: actions/setup-node@v3.7.0 + uses: actions/setup-node@v4.0.3 with: node-version-file: .nvmrc cache: 'npm'