chore(release): pull hotfix-release/v1.75.0 into main (#3649)

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.75.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.1...v1.75.0) (2024-08-12)
+
+
+### Features
+
+* move hubspot to transformer proxy to enable partial batch handling ([#3308](https://github.com/rudderlabs/rudder-transformer/issues/3308)) ([8450021](https://github.com/rudderlabs/rudder-transformer/commit/8450021672c51ac798ec0aeab422f5fceea5e53e))
+
+
+### Bug Fixes
+
+* handle attentive tag null, undefined properties ([#3647](https://github.com/rudderlabs/rudder-transformer/issues/3647)) ([9327925](https://github.com/rudderlabs/rudder-transformer/commit/932792561c98833baf9881a83ee36ae5000e37b4))
+* handle null values for braze dedupe ([#3638](https://github.com/rudderlabs/rudder-transformer/issues/3638)) ([0a9b681](https://github.com/rudderlabs/rudder-transformer/commit/0a9b68118241158623d45ee28896377296bafd91))
+
 ### [1.74.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.0...v1.74.1) (2024-08-08)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.74.1",
+  "version": "1.75.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/v0/destinations/attentive_tag/util.js

@@ -19,11 +19,13 @@ const { mappingConfig, ConfigCategory } = require('./config');
  */
 const getPropertiesKeyValidation = (payload) => {
   const validationArray = [`'`, `"`, `{`, `}`, `[`, `]`, ',', `,`];
-  const keys = Object.keys(payload.properties);
-  for (const key of keys) {
-    for (const validationChar of validationArray) {
-      if (key.includes(validationChar)) {
-        return false;
+  if (payload.properties) {
+    const keys = Object.keys(payload.properties);
+    for (const key of keys) {
+      for (const validationChar of validationArray) {
+        if (key.includes(validationChar)) {
+          return false;
+        }
       }
     }
   }

src/v0/destinations/braze/braze.util.test.js

@@ -785,6 +785,136 @@ describe('dedup utility tests', () => {
       const result = BrazeDedupUtility.deduplicate(userData, store);
       expect(result).toBeNull();
     });
+
+    test('deduplicates user data correctly when user data is null and it doesnt exist in stored data', () => {
+      const userData = {
+        external_id: '123',
+        nullProperty: null,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is null and it is same in stored data', () => {
+      const userData = {
+        external_id: '123',
+        nullProperty: null,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+          nullProperty: null,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is null and it is different in stored data', () => {
+      const userData = {
+        external_id: '123',
+        nullProperty: null,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+          nullProperty: 'valid data',
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual({
+        external_id: '123',
+        nullProperty: null,
+      });
+    });
+
+    test('deduplicates user data correctly when user data is undefined and it doesnt exist in stored data', () => {
+      const userData = {
+        external_id: '123',
+        undefinedProperty: undefined,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is undefined and it is same in stored data', () => {
+      const userData = {
+        external_id: '123',
+        undefinedProperty: undefined,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          undefinedProperty: undefined,
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is undefined and it is defined in stored data', () => {
+      const userData = {
+        external_id: '123',
+        undefinedProperty: undefined,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          undefinedProperty: 'defined data',
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual({
+        external_id: '123',
+        undefinedProperty: undefined,
+      });
+    });
   });
 });
 

src/v0/destinations/braze/util.js

@@ -288,7 +288,14 @@ const BrazeDedupUtility = {
 
     if (keys.length > 0) {
       keys.forEach((key) => {
-        if (!_.isEqual(userData[key], storedUserData[key])) {
+        // ref: https://www.braze.com/docs/user_guide/data_and_analytics/custom_data/custom_attributes/#adding-descriptions
+        // null is a valid value in braze for unsetting, so we need to compare the values only if the key is present in the stored user data
+        // in case of keys having null values only compare if the key is present in the stored user data
+        if (userData[key] === null) {
+          if (isDefinedAndNotNull(storedUserData[key])) {
+            deduplicatedUserData[key] = userData[key];
+          }
+        } else if (!_.isEqual(userData[key], storedUserData[key])) {
           deduplicatedUserData[key] = userData[key];
         }
       });

src/v0/destinations/hs/transform.js

@@ -19,6 +19,7 @@ const {
   fetchFinalSetOfTraits,
   getProperties,
   validateDestinationConfig,
+  convertToResponseFormat,
 } = require('./util');
 
 const processSingleMessage = async ({ message, destination, metadata }, propertyMap) => {
@@ -147,20 +148,38 @@ const processBatchRouter = async (inputs, reqMetadata) => {
     }),
   );
 
-  if (successRespList.length > 0) {
+  const dontBatchTrueResponses = [];
+  const dontBatchFalseOrUndefinedResponses = [];
+  // segregating successRepList depending on dontbatch value
+  successRespList.forEach((successResp) => {
+    if (successResp.metadata?.dontBatch) {
+      dontBatchTrueResponses.push(successResp);
+    } else {
+      dontBatchFalseOrUndefinedResponses.push(successResp);
+    }
+  });
+
+  // batch implementation
+  if (dontBatchFalseOrUndefinedResponses.length > 0) {
     if (destination.Config.apiVersion === API_VERSION.v3) {
-      batchedResponseList = batchEvents(successRespList);
+      batchedResponseList = batchEvents(dontBatchFalseOrUndefinedResponses);
     } else {
-      batchedResponseList = legacyBatchEvents(successRespList);
+      batchedResponseList = legacyBatchEvents(dontBatchFalseOrUndefinedResponses);
     }
   }
-  return { batchedResponseList, errorRespList };
+  return {
+    batchedResponseList,
+    errorRespList,
+    // if there are any events where dontbatch set to true we need to update them according to the response format
+    dontBatchEvents: convertToResponseFormat(dontBatchTrueResponses),
+  };
 };
 // we are batching by default at routerTransform
 const processRouterDest = async (inputs, reqMetadata) => {
   const tempNewInputs = batchEventsInOrder(inputs);
   const batchedResponseList = [];
   const errorRespList = [];
+  const dontBatchEvents = [];
   const promises = tempNewInputs.map(async (inputEvents) => {
     const response = await processBatchRouter(inputEvents, reqMetadata);
     return response;
@@ -171,8 +190,10 @@ const processRouterDest = async (inputs, reqMetadata) => {
   results.forEach((response) => {
     errorRespList.push(...response.errorRespList);
     batchedResponseList.push(...response.batchedResponseList);
+    dontBatchEvents.push(...response.dontBatchEvents);
   });
-  return [...batchedResponseList, ...errorRespList];
+  console.log(JSON.stringify([...batchedResponseList, ...errorRespList, ...dontBatchEvents]));
+  return [...batchedResponseList, ...errorRespList, ...dontBatchEvents];
 };
 
 module.exports = { process, processRouterDest };

src/v0/destinations/hs/util.js

@@ -22,6 +22,9 @@ const {
   getValueFromMessage,
   isNull,
   validateEventName,
+  defaultBatchRequestConfig,
+  defaultPostRequestConfig,
+  getSuccessRespEvents,
 } = require('../../util');
 const {
   CONTACT_PROPERTY_MAP_ENDPOINT,
@@ -844,6 +847,34 @@ const addExternalIdToHSTraits = (message) => {
   set(getFieldValueFromMessage(message, 'traits'), externalIdObj.identifierType, externalIdObj.id);
 };
 
+const convertToResponseFormat = (successRespListWithDontBatchTrue) => {
+  const response = [];
+  if (Array.isArray(successRespListWithDontBatchTrue)) {
+    successRespListWithDontBatchTrue.forEach((event) => {
+      const { message, metadata, destination } = event;
+      const endpoint = get(message, 'endpoint');
+
+      const batchedResponse = defaultBatchRequestConfig();
+      batchedResponse.batchedRequest.headers = message.headers;
+      batchedResponse.batchedRequest.endpoint = endpoint;
+      batchedResponse.batchedRequest.body = message.body;
+      batchedResponse.batchedRequest.params = message.params;
+      batchedResponse.batchedRequest.method = defaultPostRequestConfig.requestMethod;
+      batchedResponse.metadata = [metadata];
+      batchedResponse.destination = destination;
+
+      response.push(
+        getSuccessRespEvents(
+          batchedResponse.batchedRequest,
+          batchedResponse.metadata,
+          batchedResponse.destination,
+        ),
+      );
+    });
+  }
+  return response;
+};
+
 module.exports = {
   validateDestinationConfig,
   addExternalIdToHSTraits,
@@ -863,4 +894,5 @@ module.exports = {
   getObjectAndIdentifierType,
   extractIDsForSearchAPI,
   getRequestData,
+  convertToResponseFormat,
 };