From 2907468e9faf11628406b9470d50c3c025cd4643 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 18 Jan 2024 12:35:27 +0530 Subject: [PATCH 1/3] fix: mailjet source operating on array instead object --- src/v0/sources/mailjet/transform.js | 84 ++++---- test/__tests__/data/mailjet_source.json | 258 +++++++++++++++++------- test/__tests__/mailjet_source.test.js | 2 +- 3 files changed, 234 insertions(+), 110 deletions(-) diff --git a/src/v0/sources/mailjet/transform.js b/src/v0/sources/mailjet/transform.js index d2719cc0ba..3c96834160 100644 --- a/src/v0/sources/mailjet/transform.js +++ b/src/v0/sources/mailjet/transform.js @@ -7,45 +7,49 @@ const Message = require('../message'); // import mapping json using JSON.parse to preserve object key order const mapping = JSON.parse(fs.readFileSync(path.resolve(__dirname, './mapping.json'), 'utf-8')); -function process(event) { - const message = new Message(`MailJet`); - - // event type is always track - const eventType = 'track'; - - message.setEventType(eventType); - - message.setEventName(event.event); - - message.setPropertiesV2(event, mapping); - - if (event.time) { - const ts = new Date(event.time * 1000).toISOString(); - message.setProperty('originalTimestamp', ts); - } - - const externalId = []; - // setting up mailjet contact_id and list_id to externalId - if (event.mj_contact_id) { - externalId.push({ - type: 'mailjetContactId', - id: event.mj_contact_id, - }); - } - if (event.mj_list_id) { - externalId.push({ - type: 'mailjetListId', - id: event.mj_list_id, - }); - } - message.context.externalId = externalId; - - if (!message.userId && event.email) { - // Treating userId as unique identifier - // If userId is not present, then generating it from email using md5 hash function - message.userId = md5(event.email); - } - return message; -} +const processEvent = (eventList) => { + const messages = []; + eventList.forEach((event) => { + const message = new Message(`MailJet`); + // event type is always track + const eventType = 'track'; + message.setEventType(eventType); + message.setEventName(event.event); + message.setPropertiesV2(event, mapping); + + if (event.time) { + const ts = new Date(event.time * 1000).toISOString(); + message.setProperty('originalTimestamp', ts); + } + + const externalId = []; + // setting up mailjet contact_id and list_id to externalId + if (event.mj_contact_id) { + externalId.push({ + type: 'mailjetContactId', + id: event.mj_contact_id, + }); + } + if (event.mj_list_id) { + externalId.push({ + type: 'mailjetListId', + id: event.mj_list_id, + }); + } + message.context.externalId = externalId; + + if (!message.userId && event.email) { + // Treating userId as unique identifier + // If userId is not present, then generating it from email using md5 hash function + message.userId = md5(event.email); + } + messages.push(message); + }); + + return messages; +}; + +// This fucntion just converts the incoming payload to array of already not and sends it to processEvent +const process = (event) => (Array.isArray(event) ? processEvent(event) : processEvent([event])); module.exports = { process }; diff --git a/test/__tests__/data/mailjet_source.json b/test/__tests__/data/mailjet_source.json index d9869585cb..6d9562e2c4 100644 --- a/test/__tests__/data/mailjet_source.json +++ b/test/__tests__/data/mailjet_source.json @@ -1,23 +1,25 @@ [ { "description": "MailJet email open event", - "input": { - "event": "open", - "time": 1664443614, - "MessageID": 94857068804950690, - "Message_GUID": "54d6cdec-f659-4547-8926-13d9c4126b82", - "email": "test@rudderstack.com", - "mj_campaign_id": 108760, - "mj_contact_id": 399962859, - "customcampaign": "mj.nl=58424", - "ip": "66.249.84.231", - "geo": "US", - "agent": "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)", - "CustomID": "", - "Payload": "" - }, - "output": { - "Message": { + "input": [ + { + "event": "open", + "time": 1664443614, + "MessageID": 94857068804950690, + "Message_GUID": "54d6cdec-f659-4547-8926-13d9c4126b82", + "email": "test@rudderstack.com", + "mj_campaign_id": 108760, + "mj_contact_id": 399962859, + "customcampaign": "mj.nl=58424", + "ip": "66.249.84.231", + "geo": "US", + "agent": "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)", + "CustomID": "", + "Payload": "" + } + ], + "output": [ + { "context": { "library": { "name": "unknown", @@ -52,10 +54,10 @@ "originalTimestamp": "2022-09-29T09:26:54.000Z", "userId": "5b6a3426dba2cb24e4f0aeec43bee9d7" } - } + ] }, { - "description": "MailJet email bounce event", + "description": "MailJet email bounce event where input event is of type ", "input": { "event": "bounce", "time": 1664444171, @@ -70,8 +72,8 @@ "error_related_to": "system", "error": "connection issue" }, - "output": { - "Message": { + "output": [ + { "context": { "library": { "name": "unknown", @@ -102,23 +104,25 @@ "originalTimestamp": "2022-09-29T09:36:11.000Z", "userId": "5b6a3426dba2cb24e4f0aeec43bee9d7" } - } + ] }, { "description": "MailJet email sent event", - "input": { - "event": "sent", - "time": 1664444171, - "MessageID": 92886743924596480, - "Message_GUID": "0230c73a-2b77-4aea-8ef2-ed15d0edc5fd", - "email": "test@rudderstack.com", - "mj_campaign_id": 108892, - "mj_contact_id": 372651182, - "customcampaign": "mj.nl=58486", - "smtp_reply": "250 2.0.0 OK DMARC:Quarantine 1664444171 u17-20020adfdd51000000b0022cc3f2bf13si3225188wrm.271 - gsmtp" - }, - "output": { - "Message": { + "input": [ + { + "event": "sent", + "time": 1664444171, + "MessageID": 92886743924596480, + "Message_GUID": "0230c73a-2b77-4aea-8ef2-ed15d0edc5fd", + "email": "test@rudderstack.com", + "mj_campaign_id": 108892, + "mj_contact_id": 372651182, + "customcampaign": "mj.nl=58486", + "smtp_reply": "250 2.0.0 OK DMARC:Quarantine 1664444171 u17-20020adfdd51000000b0022cc3f2bf13si3225188wrm.271 - gsmtp" + } + ], + "output": [ + { "context": { "library": { "name": "unknown", @@ -149,26 +153,28 @@ "originalTimestamp": "2022-09-29T09:36:11.000Z", "userId": "5b6a3426dba2cb24e4f0aeec43bee9d7" } - } + ] }, { "description": "MailJet email bounce event", - "input": { - "event": "bounce", - "time": 1664444170, - "MessageID": 56013522696710744, - "Message_GUID": "dbe4f0a3-4a5a-4784-a724-a9794d3c0444", - "email": "test@rudderstack.com", - "mj_campaign_id": 108892, - "mj_contact_id": 373142182, - "customcampaign": "mj.nl=58486", - "blocked": false, - "hard_bounce": false, - "error_related_to": "system", - "error": "connection issue" - }, - "output": { - "Message": { + "input": [ + { + "event": "bounce", + "time": 1664444170, + "MessageID": 56013522696710744, + "Message_GUID": "dbe4f0a3-4a5a-4784-a724-a9794d3c0444", + "email": "test@rudderstack.com", + "mj_campaign_id": 108892, + "mj_contact_id": 373142182, + "customcampaign": "mj.nl=58486", + "blocked": false, + "hard_bounce": false, + "error_related_to": "system", + "error": "connection issue" + } + ], + "output": [ + { "context": { "library": { "name": "unknown", @@ -199,25 +205,27 @@ "originalTimestamp": "2022-09-29T09:36:10.000Z", "userId": "5b6a3426dba2cb24e4f0aeec43bee9d7" } - } + ] }, { "description": "MailJet when no email is present", - "input": { - "event": "bounce", - "time": 1664444170, - "MessageID": 56013522696710744, - "Message_GUID": "dbe4f0a3-4a5a-4784-a724-a9794d3c0444", - "mj_campaign_id": 108892, - "mj_contact_id": 373142182, - "customcampaign": "mj.nl=58486", - "blocked": false, - "hard_bounce": false, - "error_related_to": "system", - "error": "connection issue" - }, - "output": { - "Message": { + "input": [ + { + "event": "bounce", + "time": 1664444170, + "MessageID": 56013522696710744, + "Message_GUID": "dbe4f0a3-4a5a-4784-a724-a9794d3c0444", + "mj_campaign_id": 108892, + "mj_contact_id": 373142182, + "customcampaign": "mj.nl=58486", + "blocked": false, + "hard_bounce": false, + "error_related_to": "system", + "error": "connection issue" + } + ], + "output": [ + { "context": { "library": { "name": "unknown", @@ -244,6 +252,118 @@ }, "originalTimestamp": "2022-09-29T09:36:10.000Z" } - } + ] + }, + { + "description": "MailJet Multiple payloads in single request", + "input": [ + { + "event": "open", + "time": 1704458040, + "MessageID": 987654, + "Message_GUID": "876r-oihugyf-7tfygh", + "email": "abc@r.com", + "mj_campaign_id": 321, + "mj_contact_id": 123, + "customcampaign": "test_campaign", + "url": "https://www.example.com/", + "ip": "ip_info", + "geo": "some geo info", + "agent": "mailjet api test" + }, + { + "event": "click", + "time": 1704458041, + "MessageID": 12345234567, + "Message_GUID": "12345-kjhgfd-2efv", + "email": "abc@r.com", + "mj_campaign_id": 12, + "mj_contact_id": 32532, + "customcampaign": "test_campaign", + "url": "https://www.example.com/", + "ip": "ip_info", + "geo": "some geo info", + "agent": "mailjet api test" + } + ], + "output": [ + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "ip": "ip_info", + "integration": { + "name": "MailJet" + }, + "traits": { + "email": "abc@r.com" + }, + "page": { + "url": "https://www.example.com/" + }, + "userAgent": "mailjet api test", + "externalId": [ + { + "type": "mailjetContactId", + "id": 123 + } + ] + }, + "integrations": { + "MailJet": false + }, + "type": "track", + "event": "open", + "properties": { + "customcampaign": "test_campaign", + "mj_campaign_id": 321, + "ip": "ip_info", + "url": "https://www.example.com/" + }, + "userId": "593a5aff0b445b3b77a6d9676b7ec86e", + "originalTimestamp": "2024-01-05T12:34:00.000Z" + }, + { + "context": { + "library": { + "name": "unknown", + "version": "unknown" + }, + "page": { + "url": "https://www.example.com/" + }, + + "integration": { + "name": "MailJet" + }, + "traits": { + "email": "abc@r.com" + }, + "userAgent": "mailjet api test", + "ip": "ip_info", + "externalId": [ + { + "type": "mailjetContactId", + "id": 32532 + } + ] + }, + "integrations": { + "MailJet": false + }, + "type": "track", + "event": "click", + "properties": { + "customcampaign": "test_campaign", + "mj_campaign_id": 12, + "ip": "ip_info", + "url": "https://www.example.com/" + }, + "userId": "593a5aff0b445b3b77a6d9676b7ec86e", + "originalTimestamp": "2024-01-05T12:34:01.000Z" + } + ] } ] diff --git a/test/__tests__/mailjet_source.test.js b/test/__tests__/mailjet_source.test.js index 9081d21122..7a778e7b4e 100644 --- a/test/__tests__/mailjet_source.test.js +++ b/test/__tests__/mailjet_source.test.js @@ -18,7 +18,7 @@ testData.forEach((data, index) => { it(`${name} Tests: payload: ${index}`, () => { try { const output = transformer.process(data.input); - expect(output).toEqual(data.output.Message); + expect(output).toEqual(data.output); } catch (error) { expect(error.message).toEqual(data.output); } From ec48cb11477e22549ec164cc96e3112f8914bbdb Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 18 Jan 2024 14:35:50 +0530 Subject: [PATCH 2/3] chore: small fix --- src/v0/sources/mailjet/transform.js | 81 ++++++++++++++--------------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/src/v0/sources/mailjet/transform.js b/src/v0/sources/mailjet/transform.js index 3c96834160..beb49bf8f3 100644 --- a/src/v0/sources/mailjet/transform.js +++ b/src/v0/sources/mailjet/transform.js @@ -1,3 +1,4 @@ +const workflowEngine1 = require('@rudderstack/workflow-engine'); const path = require('path'); const fs = require('fs'); const md5 = require('md5'); @@ -7,49 +8,47 @@ const Message = require('../message'); // import mapping json using JSON.parse to preserve object key order const mapping = JSON.parse(fs.readFileSync(path.resolve(__dirname, './mapping.json'), 'utf-8')); -const processEvent = (eventList) => { - const messages = []; - eventList.forEach((event) => { - const message = new Message(`MailJet`); - // event type is always track - const eventType = 'track'; - message.setEventType(eventType); - message.setEventName(event.event); - message.setPropertiesV2(event, mapping); - - if (event.time) { - const ts = new Date(event.time * 1000).toISOString(); - message.setProperty('originalTimestamp', ts); - } - - const externalId = []; - // setting up mailjet contact_id and list_id to externalId - if (event.mj_contact_id) { - externalId.push({ - type: 'mailjetContactId', - id: event.mj_contact_id, - }); - } - if (event.mj_list_id) { - externalId.push({ - type: 'mailjetListId', - id: event.mj_list_id, - }); - } - message.context.externalId = externalId; - - if (!message.userId && event.email) { - // Treating userId as unique identifier - // If userId is not present, then generating it from email using md5 hash function - message.userId = md5(event.email); - } - messages.push(message); - }); - - return messages; +const processEvent = (event) => { + const message = new Message(`MailJet`); + // event type is always track + const eventType = 'track'; + message.setEventType(eventType); + message.setEventName(event.event); + message.setPropertiesV2(event, mapping); + + if (event.time) { + const ts = new Date(event.time * 1000).toISOString(); + message.setProperty('originalTimestamp', ts); + } + + const externalId = []; + // setting up mailjet contact_id and list_id to externalId + if (event.mj_contact_id) { + externalId.push({ + type: 'mailjetContactId', + id: event.mj_contact_id, + }); + } + if (event.mj_list_id) { + externalId.push({ + type: 'mailjetListId', + id: event.mj_list_id, + }); + } + message.context.externalId = externalId; + + if (!message.userId && event.email) { + // Treating userId as unique identifier + // If userId is not present, then generating it from email using md5 hash function + message.userId = md5(event.email); + } + return message; }; // This fucntion just converts the incoming payload to array of already not and sends it to processEvent -const process = (event) => (Array.isArray(event) ? processEvent(event) : processEvent([event])); +const process = (events) => { + const eventsArray = workflowEngine1.toArray(events); + return eventsArray.map(processEvent); +}; module.exports = { process }; From f8ed44a20684888b3e44293265171e9cc8dc2171 Mon Sep 17 00:00:00 2001 From: Dilip Kola Date: Thu, 18 Jan 2024 15:23:01 +0530 Subject: [PATCH 3/3] refactor: use toArray from transformer utils --- src/v0/sources/mailjet/transform.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v0/sources/mailjet/transform.js b/src/v0/sources/mailjet/transform.js index beb49bf8f3..91f90b6ed0 100644 --- a/src/v0/sources/mailjet/transform.js +++ b/src/v0/sources/mailjet/transform.js @@ -1,8 +1,8 @@ -const workflowEngine1 = require('@rudderstack/workflow-engine'); const path = require('path'); const fs = require('fs'); const md5 = require('md5'); const Message = require('../message'); +const { CommonUtils } = require('../../../util/common'); // ref : https://dev.mailjet.com/email/guides/webhooks/ // import mapping json using JSON.parse to preserve object key order @@ -47,7 +47,7 @@ const processEvent = (event) => { // This fucntion just converts the incoming payload to array of already not and sends it to processEvent const process = (events) => { - const eventsArray = workflowEngine1.toArray(events); + const eventsArray = CommonUtils.toArray(events); return eventsArray.map(processEvent); };