fix: adding map for marketo known values and javascript known values

src/v0/destinations/marketo_bulk_upload/config.js

@@ -18,6 +18,24 @@ const FETCH_FAILURE_JOB_STATUS_ERR_MSG = 'Could not fetch failure job status';
 const FETCH_WARNING_JOB_STATUS_ERR_MSG = 'Could not fetch warning job status';
 const ACCESS_TOKEN_FETCH_ERR_MSG = 'Error during fetching access token';
 
+const SCHEMA_DATA_TYPE_MAP = {
+  string: 'string',
+  number: 'number',
+  boolean: 'boolean',
+  undefined: 'undefined',
+  float: 'number',
+  text: 'string',
+  currency: 'string',
+  integer: 'number',
+  reference: 'string',
+  datetime: 'string',
+  date: 'string',
+  email: 'string',
+  phone: 'string',
+  url: 'string',
+  object: 'object',
+};
+
 module.exports = {
   ABORTABLE_CODES,
   RETRYABLE_CODES,
@@ -33,4 +51,5 @@ module.exports = {
   FETCH_FAILURE_JOB_STATUS_ERR_MSG,
   FETCH_WARNING_JOB_STATUS_ERR_MSG,
   ACCESS_TOKEN_FETCH_ERR_MSG,
+  SCHEMA_DATA_TYPE_MAP,
 };

src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js

@@ -3,6 +3,7 @@ const {
   handlePollResponse,
   handleFileUploadResponse,
   getAccessToken,
+  checkEventStatusViaSchemaMatching,
 } = require('./util');
 
 const {
@@ -353,3 +354,187 @@ describe('getAccessToken', () => {
     await expect(getAccessToken(config)).rejects.toThrow(TransformationError);
   });
 });
+
+describe('checkEventStatusViaSchemaMatching', () => {
+  // The function correctly identifies fields with expected data types.
+  it('if event data types match with expected data types we send no field as mismatch', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            email: 'value1',
+            id: 123,
+            isLead: true,
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'string',
+      id: 'integer',
+      isLead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({});
+  });
+
+  // The function correctly identifies fields with unexpected data types.
+  it('if event data types do not match with expected data types we send that field as mismatch', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            email: 123,
+            city: '123',
+            islead: true,
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'string',
+      city: 'number',
+      islead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({
+      job1: 'invalid email for value 123 ',
+    });
+  });
+
+  // The function correctly handles events with multiple fields.
+  it('For array of events the mismatch object fills up with each event errors', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            id: 'value1',
+            testCustomFieldScore: 123,
+            isLead: true,
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+        {
+          message: {
+            email: 'value2',
+            id: 456,
+            testCustomFieldScore: false,
+          },
+          metadata: {
+            job_id: 'job2',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'email',
+      id: 'integer',
+      testCustomFieldScore: 'integer',
+      isLead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({
+      job1: 'invalid id for value value1 ',
+      job2: 'invalid testCustomFieldScore for value false ',
+    });
+  });
+
+  // The function correctly handles events with missing fields.
+  it('it is not mandatory to send all the fields present in schema', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            email: 'value1',
+            isLead: true,
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'string',
+      id: 'number',
+      isLead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({});
+  });
+
+  // The function correctly handles events with additional fields. But this will not happen in our use case
+  it('for any field beyond schema fields will be mapped as invalid', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            email: 'value1',
+            id: 124,
+            isLead: true,
+            abc: 'value2',
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'string',
+      id: 'number',
+      isLead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({
+      job1: 'invalid abc for value value2 ',
+    });
+  });
+
+  // The function correctly handles events with null values.
+  it('should correctly handle events with null values', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            email: 'value1',
+            id: null,
+            isLead: true,
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'string',
+      id: 'number',
+      isLead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({
+      job1: 'invalid id for value null ',
+    });
+  });
+});

src/v0/destinations/marketo_bulk_upload/util.js

@@ -18,6 +18,7 @@ const {
   POLL_STATUS_ERR_MSG,
   FILE_UPLOAD_ERR_MSG,
   ACCESS_TOKEN_FETCH_ERR_MSG,
+  SCHEMA_DATA_TYPE_MAP,
 } = require('./config');
 const logger = require('../../../logger');
 
@@ -401,16 +402,11 @@ const checkEventStatusViaSchemaMatching = (event, fieldMap) => {
     const { job_id } = metadata;
 
     Object.entries(message).forEach(([paramName, paramValue]) => {
-      let expectedDataType = fieldMap[paramName];
+      const expectedDataType = SCHEMA_DATA_TYPE_MAP[fieldMap[paramName]];
       const actualDataType = typeof paramValue;
 
-      // If expectedDataType is not one of the primitive data types, treat it as a string
-      if (!['string', 'number', 'boolean', 'undefined'].includes(expectedDataType)) {
-        expectedDataType = 'string';
-      }
-
       if (!mismatchedFields[job_id] && actualDataType !== expectedDataType) {
-        mismatchedFields[job_id] = `invalid ${paramName}`;
+        mismatchedFields[job_id] = `invalid ${paramName} for value ${paramValue} `;
       }
     });
   });