Merge branch 'develop' into fix_improper_type_validations_snyc

.github/pull_request_template.md

@@ -2,21 +2,15 @@
 
 Write a brief explainer on your code changes.
 
-## Please explain the objectives of your changes below
-
-Put down any required details on the broader aspect of your changes. If there are any dependent changes, **mandatorily** mention them here
-
-### Type of change
+## What is the related Linear task?
 
-If the pull request is a **bug-fix**, **enhancement** or a **refactor**, please fill in the details on the changes made.
+Resolves INT-XXX
 
-- Existing capabilities/behavior
-
-- New capabilities/behavior
+## Please explain the objectives of your changes below
 
-If the pull request is a **new feature**,
+Put down any required details on the broader aspect of your changes. If there are any dependent changes, **mandatorily** mention them here
 
-### Any technical or performance related pointers to consider with the change?
+### Any changes to existing capabilities/behaviour, mention the reason & what are the changes ?
 
 N/A
 
@@ -28,25 +22,27 @@ N/A
 
 N/A
 
-### If the PR has changes in more than 10 files, please mention why the changes were not split into multiple PRs.
-
-N/A
-
-### If multiple linear tasks are associated with the PR changes, please elaborate on the reason:
+### Any technical or performance related pointers to consider with the change?
 
 N/A
 
 <hr>
 
 ### Developer checklist
 
+- [ ] My code follows the style guidelines of this project
+
 - [ ] **No breaking changes are being introduced.**
 
-- [ ] Are all related docs linked with the PR?
+- [ ] All related docs linked with the PR?
+
+- [ ] All changes manually tested?
+
+- [ ] Any documentation changes needed with this change?
 
-- [ ] Are all changes manually tested?
+- [ ] Is the PR limited to 10 file changes?
 
-- [ ] Does this change require any documentation changes?
+- [ ] Is the PR limited to one linear task?
 
 - [ ] Are relevant unit and component test-cases added?
 

.nvmrc

@@ -1 +1 @@
-18.16.0
+18.19.0

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.52.4](https://github.com/rudderlabs/rudder-transformer/compare/v1.52.3...v1.52.4) (2023-12-27)
+
+
+### Bug Fixes
+
+* send empty string instead of null when the schema data is undefined ([#2955](https://github.com/rudderlabs/rudder-transformer/issues/2955)) ([511741e](https://github.com/rudderlabs/rudder-transformer/commit/511741ed356b365f52e0335ce6a1fc953ccbc465))
+
 ### [1.52.3](https://github.com/rudderlabs/rudder-transformer/compare/v1.52.2...v1.52.3) (2023-12-18)
 
 

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.4
-FROM node:18.17-alpine3.17 AS base
+FROM node:18.19.0-alpine3.18 AS base
 ENV HUSKY 0
 
 RUN apk update

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.52.3",
+  "version": "1.52.4",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/v0/destinations/am/transform.js

@@ -297,6 +297,15 @@ const identifyBuilder = (message, destination, rawPayload) => {
       }
     });
   }
+  // update identify call request with unset fields
+  // AM docs https://www.docs.developers.amplitude.com/analytics/apis/http-v2-api/#keys-for-the-event-argument:~:text=exceed%2040%20layers.-,user_properties,-Optional.%20Object.%20A
+  const unsetObject = AMUtils.getUnsetObj(message);
+  if (unsetObject) {
+    // Example   unsetObject = {
+    //     "testObj.del1": "-"
+    // }
+    set(rawPayload, `user_properties.$unset`, unsetObject);
+  }
   return rawPayload;
 };
 
@@ -334,7 +343,7 @@ const getResponseData = (evType, destination, rawPayload, message, groupInfo) =>
     case EventType.IDENTIFY:
       // event_type for identify event is $identify
       rawPayload.event_type = IDENTIFY_AM;
-      identifyBuilder(message, destination, rawPayload);
+      rawPayload = identifyBuilder(message, destination, rawPayload);
       break;
     case EventType.GROUP:
       // event_type for identify event is $identify

src/v0/destinations/am/util.test.js

@@ -0,0 +1,66 @@
+const { getUnsetObj } = require('./utils');
+
+describe('getUnsetObj', () => {
+  it("should return undefined when 'message.integrations.Amplitude.fieldsToUnset' is not array", () => {
+    const message = {
+      integrations: {
+        Amplitude: { fieldsToUnset: 'field_name' },
+      },
+    };
+    const result = getUnsetObj(message);
+    expect(result).toBeUndefined();
+  });
+  it("should return undefined when 'message.integrations.Amplitude.fieldsToUnset' is undefined", () => {
+    const message = {
+      integrations: {
+        Amplitude: {},
+      },
+    };
+    const result = getUnsetObj(message);
+    expect(result).toBeUndefined();
+  });
+
+  it("should return an empty objecty when 'message.integrations.Amplitude.fieldsToUnset' is an empty array", () => {
+    const message = {
+      integrations: {
+        Amplitude: { fieldsToUnset: [] },
+      },
+    };
+    const result = getUnsetObj(message);
+    expect(result).toEqual({});
+  });
+
+  it("should return an object with keys and values set to '-' when 'message.integrations.Amplitude.fieldsToUnset' is an array of strings", () => {
+    const message = {
+      integrations: {
+        Amplitude: { fieldsToUnset: ['Unset1', 'Unset2'] },
+      },
+    };
+    const result = getUnsetObj(message);
+    expect(result).toEqual({
+      Unset1: '-',
+      Unset2: '-',
+    });
+  });
+
+  it("should handle missing 'message' parameter", () => {
+    const result = getUnsetObj();
+    expect(result).toBeUndefined();
+  });
+
+  // Should handle missing 'integrations' property in 'message' parameter
+  it("should handle missing 'integrations' property in 'message' parameter", () => {
+    const message = {};
+    const result = getUnsetObj(message);
+    expect(result).toBeUndefined();
+  });
+
+  // Should handle missing 'Amplitude' property in 'message.integrations' parameter
+  it("should handle missing 'Amplitude' property in 'message.integrations' parameter", () => {
+    const message = {
+      integrations: {},
+    };
+    const result = getUnsetObj(message);
+    expect(result).toBeUndefined();
+  });
+});

src/v0/destinations/am/utils.js

@@ -82,6 +82,32 @@ const getEventId = (payload, sourceKey) => {
   return undefined;
 };
 
+/**
+ * generates the unsetObject and returns it 
+ * @param {*} message 
+ * @returns 
+ * 
+ * Example message = {
+    integrations: {
+      Amplitude: { fieldsToUnset: ['Unset1', 'Unset2'] },
+      All: true,
+    },
+  };
+  return unsetObj = {
+        "Unset1": "-",
+        "Unset2": "-"
+      }
+  AM docs: https://www.docs.developers.amplitude.com/analytics/apis/http-v2-api/#keys-for-the-event-argument:~:text=exceed%2040%20layers.-,user_properties,-Optional.%20Object.%20A
+ */
+const getUnsetObj = (message) => {
+  const fieldsToUnset = get(message, 'integrations.Amplitude.fieldsToUnset');
+  let unsetObject;
+  if (Array.isArray(fieldsToUnset)) {
+    unsetObject = Object.fromEntries(fieldsToUnset.map((field) => [field, '-']));
+  }
+
+  return unsetObject;
+};
 module.exports = {
   getOSName,
   getOSVersion,
@@ -90,4 +116,5 @@ module.exports = {
   getPlatform,
   getBrand,
   getEventId,
+  getUnsetObj,
 };

src/v0/destinations/factorsai/data/FactorsAIGroupConfig.json

@@ -34,5 +34,10 @@
     "destKey": "email",
     "sourceKeys": "email",
     "required": false
+  },
+  {
+    "sourceKeys": "context",
+    "destKey": "context",
+    "required": false
   }
 ]

src/v0/destinations/factorsai/data/FactorsAIIdentifyConfig.json

@@ -34,5 +34,10 @@
     "destKey": "channel",
     "sourceKeys": "channel",
     "required": false
+  },
+  {
+    "sourceKeys": "context",
+    "destKey": "context",
+    "required": false
   }
 ]

src/v0/destinations/factorsai/data/FactorsAITrackConfig.json

@@ -44,5 +44,10 @@
     "destKey": "channel",
     "sourceKeys": "channel",
     "required": false
+  },
+  {
+    "sourceKeys": "context",
+    "destKey": "context",
+    "required": false
   }
 ]

src/v0/destinations/factorsai/transform.js

@@ -13,6 +13,14 @@ const { JSON_MIME_TYPE } = require('../../util/constant');
 
 const { ConfigCategories, mappingConfig, BASE_URL } = require('./config');
 
+function populateIpDetails(requestJson, message) {
+  const payload = requestJson;
+  if (message.context || message.request_ip) {
+    payload.context = { ...(payload.context || {}), ip: message.context?.ip || message.request_ip };
+  }
+  return payload;
+}
+
 // build final response
 function buildResponse(payload, factorsAIApiKey) {
   const response = defaultRequestConfig();
@@ -29,21 +37,24 @@ function buildResponse(payload, factorsAIApiKey) {
 
 // process identify call
 function processIdentify(message, factorsAIApiKey) {
-  const requestJson = constructPayload(message, mappingConfig[ConfigCategories.IDENTIFY.name]);
+  let requestJson = constructPayload(message, mappingConfig[ConfigCategories.IDENTIFY.name]);
+  requestJson = populateIpDetails(requestJson, message);
   return buildResponse(requestJson, factorsAIApiKey);
 }
 
 // process track call
 function processTrack(message, factorsAIApiKey) {
-  const requestJson = constructPayload(message, mappingConfig[ConfigCategories.TRACK.name]);
+  let requestJson = constructPayload(message, mappingConfig[ConfigCategories.TRACK.name]);
+  requestJson = populateIpDetails(requestJson, message);
   // flatten json as factorsAi do not support nested properties
   requestJson.properties = flattenJson(requestJson.properties);
   return buildResponse(requestJson, factorsAIApiKey);
 }
 
 // process Page Call
 function processPageAndGroup(message, factorsAIApiKey, category) {
-  const requestJson = constructPayload(message, mappingConfig[category]);
+  let requestJson = constructPayload(message, mappingConfig[category]);
+  requestJson = populateIpDetails(requestJson, message);
   requestJson.type = message.type;
   return buildResponse(requestJson, factorsAIApiKey);
 }

src/v0/destinations/fb_custom_audience/util.js

@@ -131,21 +131,21 @@ const getUpdatedDataElement = (dataElement, isHashRequired, eachProperty, update
   /**
    * hash the original value for the properties apart from 'MADID' && 'EXTERN_ID as hashing is not required for them
    * ref: https://developers.facebook.com/docs/marketing-api/audiences/guides/custom-audiences#hash
-   * sending null values for the properties for which user hasn't provided any value
+   * sending empty string for the properties for which user hasn't provided any value
    */
   if (isHashRequired && eachProperty !== 'MADID' && eachProperty !== 'EXTERN_ID') {
     if (tmpUpdatedProperty) {
       tmpUpdatedProperty = `${tmpUpdatedProperty}`;
       dataElement.push(sha256(tmpUpdatedProperty));
     } else {
-      dataElement.push(null);
+      dataElement.push('');
     }
   }
-  // if property name is MADID or EXTERN_ID if the value is undefined send null
+  // if property name is MADID or EXTERN_ID if the value is undefined send empty string
   else if (!tmpUpdatedProperty && (eachProperty === 'MADID' || eachProperty === 'EXTERN_ID')) {
-    dataElement.push(null);
+    dataElement.push('');
   } else {
-    dataElement.push(tmpUpdatedProperty);
+    dataElement.push(tmpUpdatedProperty || '');
   }
   return dataElement;
 };

src/v0/destinations/tiktok_ads/util.js

@@ -21,6 +21,7 @@ const getContents = (message) => {
         price: product.price,
         quantity: product.quantity,
         description: product.description,
+        brand: product.brand
       };
       contents.push(removeUndefinedAndNullValues(singleProduct));
     });