From 64ab555d31b4c1c49863794444bd79b2b6a45927 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:17:52 +0530 Subject: [PATCH] fix: adding map for marketo known values and javascript known values (#3037) * fix: adding map for marketo known values and javascript known values * fix: reverting launch.json to original state * fix: editing test cases * fix: review comments addressed * fix: making error statement better * feat: reviw comments addressed --- .../marketo_bulk_upload/config.js | 19 ++ .../marketo_bulk_upload.util.test.js | 185 ++++++++++++++++++ .../destinations/marketo_bulk_upload/util.js | 8 +- 3 files changed, 206 insertions(+), 6 deletions(-) diff --git a/src/v0/destinations/marketo_bulk_upload/config.js b/src/v0/destinations/marketo_bulk_upload/config.js index 487e11fe24..e3268711fe 100644 --- a/src/v0/destinations/marketo_bulk_upload/config.js +++ b/src/v0/destinations/marketo_bulk_upload/config.js @@ -18,6 +18,24 @@ const FETCH_FAILURE_JOB_STATUS_ERR_MSG = 'Could not fetch failure job status'; const FETCH_WARNING_JOB_STATUS_ERR_MSG = 'Could not fetch warning job status'; const ACCESS_TOKEN_FETCH_ERR_MSG = 'Error during fetching access token'; +const SCHEMA_DATA_TYPE_MAP = { + string: 'string', + number: 'number', + boolean: 'boolean', + undefined: 'undefined', + float: 'number', + text: 'string', + currency: 'string', + integer: 'number', + reference: 'string', + datetime: 'string', + date: 'string', + email: 'string', + phone: 'string', + url: 'string', + object: 'object', +}; + module.exports = { ABORTABLE_CODES, RETRYABLE_CODES, @@ -33,4 +51,5 @@ module.exports = { FETCH_FAILURE_JOB_STATUS_ERR_MSG, FETCH_WARNING_JOB_STATUS_ERR_MSG, ACCESS_TOKEN_FETCH_ERR_MSG, + SCHEMA_DATA_TYPE_MAP, }; diff --git a/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js b/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js index 769fa4006d..875b0d8280 100644 --- a/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js +++ b/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js @@ -3,6 +3,7 @@ const { handlePollResponse, handleFileUploadResponse, getAccessToken, + checkEventStatusViaSchemaMatching, } = require('./util'); const { @@ -353,3 +354,187 @@ describe('getAccessToken', () => { await expect(getAccessToken(config)).rejects.toThrow(TransformationError); }); }); + +describe('checkEventStatusViaSchemaMatching', () => { + // The function correctly identifies fields with expected data types. + it('if event data types match with expected data types we send no field as mismatch', () => { + const event = { + input: [ + { + message: { + email: 'value1', + id: 123, + isLead: true, + }, + metadata: { + job_id: 'job1', + }, + }, + ], + }; + const fieldSchemaMapping = { + email: 'string', + id: 'integer', + isLead: 'boolean', + }; + + const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); + + expect(result).toEqual({}); + }); + + // The function correctly identifies fields with unexpected data types. + it('if event data types do not match with expected data types we send that field as mismatch', () => { + const event = { + input: [ + { + message: { + email: 123, + city: '123', + islead: true, + }, + metadata: { + job_id: 'job1', + }, + }, + ], + }; + const fieldSchemaMapping = { + email: 'string', + city: 'number', + islead: 'boolean', + }; + + const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); + + expect(result).toEqual({ + job1: 'invalid email', + }); + }); + + // The function correctly handles events with multiple fields. + it('For array of events the mismatch object fills up with each event errors', () => { + const event = { + input: [ + { + message: { + id: 'value1', + testCustomFieldScore: 123, + isLead: true, + }, + metadata: { + job_id: 'job1', + }, + }, + { + message: { + email: 'value2', + id: 456, + testCustomFieldScore: false, + }, + metadata: { + job_id: 'job2', + }, + }, + ], + }; + const fieldSchemaMapping = { + email: 'email', + id: 'integer', + testCustomFieldScore: 'integer', + isLead: 'boolean', + }; + + const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); + + expect(result).toEqual({ + job1: 'invalid id', + job2: 'invalid testCustomFieldScore', + }); + }); + + // The function correctly handles events with missing fields. + it('it is not mandatory to send all the fields present in schema', () => { + const event = { + input: [ + { + message: { + email: 'value1', + isLead: true, + }, + metadata: { + job_id: 'job1', + }, + }, + ], + }; + const fieldSchemaMapping = { + email: 'string', + id: 'number', + isLead: 'boolean', + }; + + const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); + + expect(result).toEqual({}); + }); + + // The function correctly handles events with additional fields. But this will not happen in our use case + it('for any field beyond schema fields will be mapped as invalid', () => { + const event = { + input: [ + { + message: { + email: 'value1', + id: 124, + isLead: true, + abc: 'value2', + }, + metadata: { + job_id: 'job1', + }, + }, + ], + }; + const fieldSchemaMapping = { + email: 'string', + id: 'number', + isLead: 'boolean', + }; + + const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); + + expect(result).toEqual({ + job1: 'invalid abc', + }); + }); + + // The function correctly handles events with null values. + it('should correctly handle events with null values', () => { + const event = { + input: [ + { + message: { + email: 'value1', + id: null, + isLead: true, + }, + metadata: { + job_id: 'job1', + }, + }, + ], + }; + const fieldSchemaMapping = { + email: 'string', + id: 'number', + isLead: 'boolean', + }; + + const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); + + expect(result).toEqual({ + job1: 'invalid id', + }); + }); +}); diff --git a/src/v0/destinations/marketo_bulk_upload/util.js b/src/v0/destinations/marketo_bulk_upload/util.js index 8b46212b87..fac04af431 100644 --- a/src/v0/destinations/marketo_bulk_upload/util.js +++ b/src/v0/destinations/marketo_bulk_upload/util.js @@ -18,6 +18,7 @@ const { POLL_STATUS_ERR_MSG, FILE_UPLOAD_ERR_MSG, ACCESS_TOKEN_FETCH_ERR_MSG, + SCHEMA_DATA_TYPE_MAP, } = require('./config'); const logger = require('../../../logger'); @@ -401,14 +402,9 @@ const checkEventStatusViaSchemaMatching = (event, fieldMap) => { const { job_id } = metadata; Object.entries(message).forEach(([paramName, paramValue]) => { - let expectedDataType = fieldMap[paramName]; + const expectedDataType = SCHEMA_DATA_TYPE_MAP[fieldMap[paramName]]; const actualDataType = typeof paramValue; - // If expectedDataType is not one of the primitive data types, treat it as a string - if (!['string', 'number', 'boolean', 'undefined'].includes(expectedDataType)) { - expectedDataType = 'string'; - } - if (!mismatchedFields[job_id] && actualDataType !== expectedDataType) { mismatchedFields[job_id] = `invalid ${paramName}`; }