fix: adding map for marketo known values and javascript known values

rudderlabs · Jan 29, 2024 · e979e4e · e979e4e
1 parent 1c6a83c
commit e979e4e
Show file tree

Hide file tree

Showing 4 changed files with 207 additions and 3 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -19,7 +19,7 @@
       "name": "Launch TS",
       "program": "${workspaceFolder}/dist/src/index.js",
       "outFiles": ["${workspaceFolder}/dist/src/**/*.js"],
-      "runtimeExecutable": "/usr/local/bin/node"
+      "runtimeExecutable": "/Users/shrouti/.nvm/versions/node/v18.19.0/bin/node"
     },
     {
       "name": "benchmark",
@@ -29,7 +29,7 @@
       "args": ["--destinations=${input:destinations}", "--feature=${input:feature}"],
       "runtimeArgs": ["--nolazy"],
       "sourceMaps": true,
-      "runtimeExecutable": "/usr/local/bin/node"
+      "runtimeExecutable": "/Users/shrouti/.nvm/versions/node/v18.19.0/bin/node"
     },
     {
       "runtimeExecutable": "/usr/local/bin/node",

diff --git a/src/v0/destinations/marketo_bulk_upload/config.js b/src/v0/destinations/marketo_bulk_upload/config.js
@@ -18,6 +18,23 @@ const FETCH_FAILURE_JOB_STATUS_ERR_MSG = 'Could not fetch failure job status';
 const FETCH_WARNING_JOB_STATUS_ERR_MSG = 'Could not fetch warning job status';
 const ACCESS_TOKEN_FETCH_ERR_MSG = 'Error during fetching access token';
 
+const SCHEMA_DATA_TYPE_MAP = {
+  string: 'string',
+  number: 'number',
+  boolean: 'boolean',
+  undefined: 'undefined',
+  float: 'number',
+  text: 'string',
+  currency: 'string',
+  integer: 'number',
+  reference: 'string',
+  datetime: 'string',
+  date: 'string',
+  email: 'string',
+  phone: 'string',
+  url: 'string',
+};
+
 module.exports = {
   ABORTABLE_CODES,
   RETRYABLE_CODES,
@@ -33,4 +50,5 @@ module.exports = {
   FETCH_FAILURE_JOB_STATUS_ERR_MSG,
   FETCH_WARNING_JOB_STATUS_ERR_MSG,
   ACCESS_TOKEN_FETCH_ERR_MSG,
+  SCHEMA_DATA_TYPE_MAP,
 };
diff --git a/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js b/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js
@@ -3,6 +3,7 @@ const {
   handlePollResponse,
   handleFileUploadResponse,
   getAccessToken,
+  checkEventStatusViaSchemaMatching,
 } = require('./util');
 
 const {
@@ -353,3 +354,187 @@ describe('getAccessToken', () => {
     await expect(getAccessToken(config)).rejects.toThrow(TransformationError);
   });
 });
+
+describe('checkEventStatusViaSchemaMatching', () => {
+  // The function correctly identifies fields with expected data types.
+  it('should correctly identify fields with expected data types', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            email: 'value1',
+            id: 123,
+            isLead: true,
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'string',
+      id: 'integer',
+      isLead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({});
+  });
+
+  // The function correctly identifies fields with unexpected data types.
+  it('should correctly identify fields with unexpected data types', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            email: 123,
+            city: '123',
+            islead: true,
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'string',
+      city: 'number',
+      islead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({
+      job1: 'invalid email',
+    });
+  });
+
+  // The function correctly handles events with multiple fields.
+  it('should correctly handle events with multiple fields', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            id: 'value1',
+            testCustomFieldScore: 123,
+            isLead: true,
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+        {
+          message: {
+            email: 'value2',
+            id: 456,
+            testCustomFieldScore: false,
+          },
+          metadata: {
+            job_id: 'job2',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'string',
+      id: 'integer',
+      testCustomFieldScore: 'integer',
+      isLead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({
+      job1: 'invalid id',
+      job2: 'invalid testCustomFieldScore',
+    });
+  });
+
+  // The function correctly handles events with missing fields.
+  it('should have no effect with missing fields', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            field1: 'value1',
+            field3: true,
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      field1: 'string',
+      field2: 'number',
+      field3: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({});
+  });
+
+  // The function correctly handles events with additional fields. But this will not happen in our use case
+  it('should correctly handle events with additional fields', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            email: 'value1',
+            id: 124,
+            isLead: true,
+            abc: 'value2',
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'string',
+      id: 'number',
+      isLead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({
+      job1: 'invalid abc',
+    });
+  });
+
+  // The function correctly handles events with null values.
+  it('should correctly handle events with null values', () => {
+    const event = {
+      input: [
+        {
+          message: {
+            email: 'value1',
+            id: null,
+            isLead: true,
+          },
+          metadata: {
+            job_id: 'job1',
+          },
+        },
+      ],
+    };
+    const fieldSchemaMapping = {
+      email: 'string',
+      id: 'number',
+      isLead: 'boolean',
+    };
+
+    const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping);
+
+    expect(result).toEqual({
+      job1: 'invalid id',
+    });
+  });
+});
diff --git a/src/v0/destinations/marketo_bulk_upload/util.js b/src/v0/destinations/marketo_bulk_upload/util.js
@@ -18,6 +18,7 @@ const {
   POLL_STATUS_ERR_MSG,
   FILE_UPLOAD_ERR_MSG,
   ACCESS_TOKEN_FETCH_ERR_MSG,
+  SCHEMA_DATA_TYPE_MAP,
 } = require('./config');
 const logger = require('../../../logger');
 
@@ -401,7 +402,7 @@ const checkEventStatusViaSchemaMatching = (event, fieldMap) => {
     const { job_id } = metadata;
 
     Object.entries(message).forEach(([paramName, paramValue]) => {
-      let expectedDataType = fieldMap[paramName];
+      let expectedDataType = SCHEMA_DATA_TYPE_MAP[fieldMap[paramName]];
       const actualDataType = typeof paramValue;
 
       // If expectedDataType is not one of the primitive data types, treat it as a string