From 3f9bfac53e3cf9021d217a364bdf724c553aaa82 Mon Sep 17 00:00:00 2001 From: Dilip Kola Date: Mon, 24 Jun 2024 11:14:03 +0530 Subject: [PATCH 1/4] chore: upgrade packages --- package-lock.json | 18 +++++++++--------- package.json | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index d2bc9b452b..2e567d7271 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,8 +20,8 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.10", - "@rudderstack/json-template-engine": "^0.14.1", - "@rudderstack/workflow-engine": "^0.8.8", + "@rudderstack/json-template-engine": "^0.15.0", + "@rudderstack/workflow-engine": "^0.8.9", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", @@ -4526,17 +4526,17 @@ } }, "node_modules/@rudderstack/json-template-engine": { - "version": "0.14.1", - "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.14.1.tgz", - "integrity": "sha512-frQ0UyN/hslA9K4cFyHLvLLTxHFhcWZRnF1ELbAHvj8AZJCblM8LZXh4A62aMmF0FbNqgsO5CeOStPqvYm3ugg==" + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.15.0.tgz", + "integrity": "sha512-RDBCn4oYvDjWhIEuV3d8QHmdTcFUyVbpBFxtRwOb1U7zSpTZ8H74/wxBG661ayN2psFh7UJYzZENju7cLWpFTw==" }, "node_modules/@rudderstack/workflow-engine": { - "version": "0.8.8", - "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.8.8.tgz", - "integrity": "sha512-6vX13wWlfQxFktq8MnAHTBpKulrjwYyZ8ZXcyugEU/BuVedtBTfjGGFsBrlKldLPuRsTZt8XuYbW8Ua7g2C5Kw==", + "version": "0.8.9", + "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.8.9.tgz", + "integrity": "sha512-4PxiXUeJ6ulhdlS7MHB6zPV6fdRVZZ0EDnl5fRbu7gDq1h4h32t5T44RwXjLbyupuu/vsPshkBNoEpo+heUBqg==", "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", - "@rudderstack/json-template-engine": "^0.14.1", + "@rudderstack/json-template-engine": "^0.15.0", "jsonata": "^2.0.5", "lodash": "^4.17.21", "object-sizeof": "^2.6.4", diff --git a/package.json b/package.json index adeee0612b..96ec42b79c 100644 --- a/package.json +++ b/package.json @@ -65,8 +65,8 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.10", - "@rudderstack/json-template-engine": "^0.14.1", - "@rudderstack/workflow-engine": "^0.8.8", + "@rudderstack/json-template-engine": "^0.15.0", + "@rudderstack/workflow-engine": "^0.8.9", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", From bba1a3b7f09e15f5a59aeed22593751d46960ebb Mon Sep 17 00:00:00 2001 From: Manish Kumar <144022547+manish339k@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:39:40 +0530 Subject: [PATCH 2/4] feat: onboard closeCRM source (#3467) * feat: onboard closeCRM source * fix: fixed timestamp issue * fix: fixed minor issues * fix: fixed payload * fix: test case --- src/v1/sources/close_crm/config.js | 3 + src/v1/sources/close_crm/transform.js | 57 +++ test/integrations/sources/close_crm/data.ts | 395 ++++++++++++++++++++ 3 files changed, 455 insertions(+) create mode 100644 src/v1/sources/close_crm/config.js create mode 100644 src/v1/sources/close_crm/transform.js create mode 100644 test/integrations/sources/close_crm/data.ts diff --git a/src/v1/sources/close_crm/config.js b/src/v1/sources/close_crm/config.js new file mode 100644 index 0000000000..410aff8b40 --- /dev/null +++ b/src/v1/sources/close_crm/config.js @@ -0,0 +1,3 @@ +const excludedFieldList = ['changed_fields', 'previous_data']; + +module.exports = { excludedFieldList }; diff --git a/src/v1/sources/close_crm/transform.js b/src/v1/sources/close_crm/transform.js new file mode 100644 index 0000000000..b597285e62 --- /dev/null +++ b/src/v1/sources/close_crm/transform.js @@ -0,0 +1,57 @@ +const moment = require('moment'); +const { + removeUndefinedAndNullValues, + removeUndefinedAndNullRecurse, + generateUUID, + formatTimeStamp, +} = require('../../../v0/util'); +const { excludedFieldList } = require('./config'); +const Message = require('../../../v0/sources/message'); + +function processEvent(inputEvent) { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { event, subscription_id } = inputEvent; + + const message = new Message('CloseCRM'); + + // Set event type track + message.setEventType('track'); + + // Set event name + const eventName = `${event.object_type} ${event.action}`; + message.setEventName(eventName); + + // Set userId + if (event.lead_id) { + message.setProperty('userId', event.lead_id); + } else { + message.setProperty('anonymousId', generateUUID()); + } + + // Set messageId + message.setProperty('messageId', event.id); + + // Set Timestamp + const timestamp = moment.utc(event.date_updated); + message.setProperty('originalTimestamp', formatTimeStamp(timestamp, 'yyyy-MM-ddTHH:mm:ss.SSSZ')); + + // Set properties + removeUndefinedAndNullRecurse(event); + message.setProperty('properties', event); + message.setProperty('properties.subscription_id', subscription_id); + + // Remove excluding fields + excludedFieldList.forEach((field) => { + delete message.properties[field]; + }); + + return message; +} + +function process(inputEvent) { + const { event } = inputEvent; + const response = processEvent(event); + return removeUndefinedAndNullValues(response); +} + +exports.process = process; diff --git a/test/integrations/sources/close_crm/data.ts b/test/integrations/sources/close_crm/data.ts new file mode 100644 index 0000000000..08b9d13a1b --- /dev/null +++ b/test/integrations/sources/close_crm/data.ts @@ -0,0 +1,395 @@ +import utils from '../../../../src/v0/util'; + +const defaultMockFns = () => { + jest.spyOn(utils, 'generateUUID').mockReturnValue('97fcd7b2-cc24-47d7-b776-057b7b199513'); +}; + +export const data = [ + { + name: 'close_crm', + description: 'lead update', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + event: { + date_created: '2019-01-15T12:48:23.395000', + meta: { + request_method: 'PUT', + request_path: '/api/v1/opportunity/object_id/', + }, + id: 'ev_123', + action: 'updated', + date_updated: '2019-01-15T12:48:23.395000', + changed_fields: [ + 'confidence', + 'date_updated', + 'status_id', + 'status_label', + 'status_type', + ], + previous_data: { + status_type: 'active', + confidence: 70, + date_updated: '2019-01-15T12:47:39.873000+00:00', + status_id: 'stat_123', + status_label: 'Active', + }, + organization_id: 'orga_123', + data: { + contact_name: 'Mr. Jones', + user_name: 'Joe Kemp', + value_period: 'one_time', + updated_by_name: 'Joe Kemp', + date_created: '2019-01-15T12:41:24.496000+00:00', + user_id: 'user_123', + updated_by: 'user_123', + value_currency: 'USD', + organization_id: 'orga_123', + status_label: 'Won', + contact_id: 'cont_123', + status_type: 'won', + created_by_name: 'Joe Kemp', + id: 'id_12', + lead_name: 'KLine', + date_lost: null, + note: '', + date_updated: '2019-01-15T12:48:23.392000+00:00', + status_id: 'stat_12', + value: 100000, + created_by: 'user_123', + value_formatted: '$1,000', + date_won: '2019-01-15', + lead_id: 'lead_123', + confidence: 100, + }, + request_id: 'req_123', + object_id: 'object_id', + user_id: 'user_123', + object_type: 'opportunity', + lead_id: 'lead_123', + }, + subscription_id: 'whsub_123', + }, + source: {}, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + name: 'unknown', + version: 'unknown', + }, + integration: { + name: 'CloseCRM', + }, + }, + integrations: { + CloseCRM: false, + }, + type: 'track', + event: 'opportunity updated', + messageId: 'ev_123', + userId: 'lead_123', + originalTimestamp: '2019-01-TuT12:48:23.395+00:00', + properties: { + date_created: '2019-01-15T12:48:23.395000', + meta: { + request_method: 'PUT', + request_path: '/api/v1/opportunity/object_id/', + }, + id: 'ev_123', + action: 'updated', + date_updated: '2019-01-15T12:48:23.395000', + organization_id: 'orga_123', + data: { + contact_name: 'Mr. Jones', + user_name: 'Joe Kemp', + value_period: 'one_time', + updated_by_name: 'Joe Kemp', + date_created: '2019-01-15T12:41:24.496000+00:00', + user_id: 'user_123', + updated_by: 'user_123', + value_currency: 'USD', + organization_id: 'orga_123', + status_label: 'Won', + contact_id: 'cont_123', + status_type: 'won', + created_by_name: 'Joe Kemp', + id: 'id_12', + lead_name: 'KLine', + note: '', + date_updated: '2019-01-15T12:48:23.392000+00:00', + status_id: 'stat_12', + value: 100000, + created_by: 'user_123', + value_formatted: '$1,000', + date_won: '2019-01-15', + lead_id: 'lead_123', + confidence: 100, + }, + request_id: 'req_123', + object_id: 'object_id', + user_id: 'user_123', + object_type: 'opportunity', + lead_id: 'lead_123', + subscription_id: 'whsub_123', + }, + }, + ], + }, + }, + ], + }, + }, + mockFns: () => { + defaultMockFns(); + }, + }, + { + name: 'close_crm', + description: 'group creation', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + subscription_id: 'whsub_123', + event: { + id: 'ev_123', + date_created: '2024-06-13T03:53:33.917000', + date_updated: '2024-06-13T03:53:33.917000', + organization_id: 'orga_123', + user_id: 'user_123', + request_id: 'req_123', + api_key_id: null, + oauth_client_id: null, + oauth_scope: null, + object_type: 'group', + object_id: 'group_123', + lead_id: null, + action: 'created', + changed_fields: [], + meta: { + request_path: '/api/v1/graphql/', + request_method: 'POST', + }, + data: { + id: 'group_123', + name: 'Test group', + members: [ + { + user_id: 'user_123', + }, + ], + }, + previous_data: {}, + }, + }, + source: {}, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + anonymousId: '97fcd7b2-cc24-47d7-b776-057b7b199513', + context: { + integration: { + name: 'CloseCRM', + }, + library: { + name: 'unknown', + version: 'unknown', + }, + }, + event: 'group created', + integrations: { + CloseCRM: false, + }, + messageId: 'ev_123', + originalTimestamp: '2024-06-ThT03:53:33.917+00:00', + properties: { + action: 'created', + data: { + id: 'group_123', + members: [ + { + user_id: 'user_123', + }, + ], + name: 'Test group', + }, + date_created: '2024-06-13T03:53:33.917000', + date_updated: '2024-06-13T03:53:33.917000', + id: 'ev_123', + meta: { + request_method: 'POST', + request_path: '/api/v1/graphql/', + }, + object_id: 'group_123', + object_type: 'group', + organization_id: 'orga_123', + request_id: 'req_123', + subscription_id: 'whsub_123', + user_id: 'user_123', + }, + type: 'track', + }, + ], + }, + }, + ], + }, + }, + mockFns: () => { + defaultMockFns(); + }, + }, + { + name: 'close_crm', + description: 'lead deletion', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + subscription_id: 'whsub_123', + event: { + id: 'ev_123', + date_created: '2024-06-14T05:16:04.138000', + date_updated: '2024-06-14T05:16:04.138000', + organization_id: 'orga_123', + user_id: 'user_123', + request_id: 'req_123', + api_key_id: 'api_123', + oauth_client_id: null, + oauth_scope: null, + object_type: 'lead', + object_id: 'lead_123', + lead_id: 'lead_123', + action: 'deleted', + changed_fields: [], + meta: { + request_path: '/api/v1/lead/lead_123/', + request_method: 'DELETE', + }, + data: {}, + previous_data: { + created_by_name: 'Rudder User', + addresses: [], + description: '', + url: null, + date_created: '2024-06-14T05:13:42.239000+00:00', + status_id: 'stat_123', + contact_ids: ['cont_123'], + id: 'lead_12', + date_updated: '2024-06-14T05:13:42.262000+00:00', + updated_by_name: 'Rudder User', + status_label: 'Potential', + name: 'test name', + display_name: 'test name', + organization_id: 'orga_123', + updated_by: 'user_123', + created_by: 'user_123', + }, + }, + }, + source: {}, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + name: 'unknown', + version: 'unknown', + }, + integration: { + name: 'CloseCRM', + }, + }, + integrations: { + CloseCRM: false, + }, + type: 'track', + event: 'lead deleted', + userId: 'lead_123', + messageId: 'ev_123', + originalTimestamp: '2024-06-FrT05:16:04.138+00:00', + properties: { + id: 'ev_123', + date_created: '2024-06-14T05:16:04.138000', + date_updated: '2024-06-14T05:16:04.138000', + organization_id: 'orga_123', + user_id: 'user_123', + request_id: 'req_123', + api_key_id: 'api_123', + object_type: 'lead', + object_id: 'lead_123', + lead_id: 'lead_123', + action: 'deleted', + meta: { + request_path: '/api/v1/lead/lead_123/', + request_method: 'DELETE', + }, + data: {}, + subscription_id: 'whsub_123', + }, + }, + ], + }, + }, + ], + }, + }, + mockFns: () => { + defaultMockFns(); + }, + }, +]; From a0f5c2f7fe2dc8bc983a35840e86655d0f92482b Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Mon, 24 Jun 2024 15:40:01 +0530 Subject: [PATCH 3/4] fix: mapping changes for aifa,andi and asid (#3465) --- .../singular/data/SINGULARAndroidEventConfig.json | 12 +++++++++--- .../singular/data/SINGULARAndroidSessionConfig.json | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/v0/destinations/singular/data/SINGULARAndroidEventConfig.json b/src/v0/destinations/singular/data/SINGULARAndroidEventConfig.json index 78a8ed20d2..f66cda7ee5 100644 --- a/src/v0/destinations/singular/data/SINGULARAndroidEventConfig.json +++ b/src/v0/destinations/singular/data/SINGULARAndroidEventConfig.json @@ -88,17 +88,23 @@ { "destKey": "aifa", "sourceKeys": "context.device.advertisingId", - "required": true + "required": false, + "metadata": { + "defaultValue": "" + } }, { "destKey": "andi", "sourceKeys": "context.device.id", - "required": true + "required": false, + "metadata": { + "defaultValue": "" + } }, { "destKey": "asid", "sourceKeys": "properties.asid", - "required": true, + "required": false, "metadata": { "defaultValue": "" } diff --git a/src/v0/destinations/singular/data/SINGULARAndroidSessionConfig.json b/src/v0/destinations/singular/data/SINGULARAndroidSessionConfig.json index ed3b410f43..e40857c61b 100644 --- a/src/v0/destinations/singular/data/SINGULARAndroidSessionConfig.json +++ b/src/v0/destinations/singular/data/SINGULARAndroidSessionConfig.json @@ -42,7 +42,7 @@ { "destKey": "asid", "sourceKeys": "properties.asid", - "required": true, + "required": false, "metadata": { "defaultValue": "" } @@ -91,12 +91,18 @@ { "destKey": "aifa", "sourceKeys": "context.device.advertisingId", - "required": true + "required": false, + "metadata": { + "defaultValue": "" + } }, { "destKey": "andi", "sourceKeys": "context.device.id", - "required": true + "required": false, + "metadata": { + "defaultValue": "" + } }, { "destKey": "utime", From 5498d0bf5aab95675edc08bbfb88a55fc90b4549 Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Tue, 25 Jun 2024 15:32:43 +0530 Subject: [PATCH 4/4] chore: empty string default endpointPath Co-authored-by: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> --- src/adapters/network.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/network.js b/src/adapters/network.js index 91d8d628fb..4f3a7a81ef 100644 --- a/src/adapters/network.js +++ b/src/adapters/network.js @@ -151,7 +151,7 @@ const httpSend = async (options, statTags = {}) => { const requestOptions = enhanceRequestOptions(options); const { url, data, method } = requestOptions; - const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`; + const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath || ''}`; logger.requestLog(`${commonMsg} request`, { metadata: statTags?.metadata, requestDetails: {