Merge branch 'develop' into feat.refiner-component-test-refactor

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.63.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.62.2...v1.63.0) (2024-04-25)
+
+
+### Features
+
+* remove redundant data from traits in hubspot ([#3310](https://github.com/rudderlabs/rudder-transformer/issues/3310)) ([4b21f13](https://github.com/rudderlabs/rudder-transformer/commit/4b21f1353d3d9a431a0d5446d019f66a543b977b))
+
 ### [1.62.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.62.1...v1.62.2) (2024-04-18)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.62.2",
+  "version": "1.63.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/cdk/v2/destinations/algolia/procWorkflow.yaml

@@ -5,7 +5,10 @@ bindings:
   - path: ../../../../v0/destinations/algolia/config
   - name: removeUndefinedAndNullValues
     path: ../../../../v0/util
+  - name: isDefinedAndNotNull
+    path: ../../../../v0/util
   - path: ../../bindings/jsontemplate
+  - path: '@rudderstack/integrations-lib'
 
 steps:
   - name: validateInput
@@ -24,6 +27,7 @@ steps:
       let eventTypeMap = $.eventTypeMapping(.destination.Config);
       let event = .message.event.trim().toLowerCase();
       let eventType = .message.properties.eventType ?? eventTypeMap[event];
+      let eventSubType = .message.properties.eventSubtype && eventType === 'conversion' && (.message.properties.eventSubtype in $.ALLOWED_EVENT_SUBTYPES) ? .message.properties.eventSubtype;
       $.assert(eventType, "eventType is mandatory for track call");
       let payload = .message.().({
         index: .properties.index,
@@ -32,12 +36,28 @@ steps:
         filters: .properties.filters,
         objectIDs: .properties.objectIds,
         positions: .properties.positions,
+        value: $.isDefinedAndNotNull(.properties.currency) ? .properties.value,
+        currency: .properties.currency,
         userToken: {{{{$.getGenericPaths("userId", "||")}}}},
         eventName: event,
-        eventType: eventType
+        eventType: eventType,
+        eventSubtype: eventSubType
       });
       $.context.payload = $.genericpayloadValidator(payload);
 
+  - name: prepareObjectDataBlock
+    condition: $.context.payload.eventType === "conversion" && $.isDefinedAndNotNull(^.message.properties.products) && Array.isArray(^.message.properties.products)
+    description: |
+      Populate list of objectData
+    template: |
+      const products = ^.message.properties.products
+      products.($.removeUndefinedAndNullValues({
+          "queryID" :  $.isDefinedAndNotNull(.queryID) ? String(.queryID) : null,
+          "price": $.isDefinedAndNotNull(.price) && $.isDefinedAndNotNull(^.message.properties.currency) ? String(.price) : null,
+          "quantity": $.isDefinedAndNotNull(.quantity)? Number(.quantity) : null,
+          "discount": $.isDefinedAndNotNull(.discount) ? String(.discount) : null
+      }))[]
+
   - name: populateProductsData
     condition: |
       .message.properties.products && 
@@ -55,11 +75,13 @@ steps:
             const products = .message.properties.products;
             const objectIDs = ~r products.objectId;
             $.context.payload.objectIDs = Array.isArray(objectIDs) ? objectIDs[:20]:$.context.payload.objectIDs;
+            $.context.payload.objectData = $.outputs.prepareObjectDataBlock
 
   - name: validateDestPayload
     template: |
       const filters = $.context.payload.filters;
       const objectIDs = $.context.payload.objectIDs;
+      const objectData = $.context.payload.objectData;
       $.assert(!(filters && objectIDs), "event can't have both objectIds and filters at the same time.");
       $.assert(filters.length || objectIDs.length, "Either filters or objectIds is required and must be non empty.");
 

src/cdk/v2/destinations/bloomreach/procWorkflow.yaml

@@ -39,7 +39,7 @@ steps:
       const userId = .message.().(
           {{{{$.getGenericPaths("userIdOnly")}}}};
       );
-      $.assert(userId ?? .message.anonymousId, "Either one of userId or anonymousId is required. Aborting");
+      $.assert(userId || .message.anonymousId, "Either one of userId or anonymousId is required. Aborting");
 
   - name: prepareIdentifyPayload
     condition: $.context.messageType === {{$.EventType.IDENTIFY}}
@@ -64,7 +64,7 @@ steps:
       - name: pageEventName
         condition: $.context.messageType === {{$.EventType.PAGE}}
         template: |
-          const category = .message.category ?? .message.properties.category;
+          const category = .message.category || .message.properties.category;
           const name = .message.name || .message.properties.name;
           const eventNameArray = ["Viewed"];
           category ? eventNameArray.push(category);
@@ -74,7 +74,7 @@ steps:
       - name: screenEventName
         condition: $.context.messageType === {{$.EventType.SCREEN}}
         template: |
-          const category = .message.category ?? .message.properties.category;
+          const category = .message.category || .message.properties.category;
           const name = .message.name || .message.properties.name;
           const eventNameArray = ["Viewed"];
           category ? eventNameArray.push(category);

src/cdk/v2/destinations/movable_ink/procWorkflow.yaml

@@ -33,7 +33,7 @@ steps:
           {{{{$.getGenericPaths("email")}}}};
       );
 
-      $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting");
+      $.assert(userId || email || .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting");
       $.validateEventPayload(.message);
 
   - name: preparePayload
@@ -50,7 +50,7 @@ steps:
       ));
       $.context.payload = {
         ...(.message),
-        userId: userId ?? email,
+        userId: userId || email,
         timestamp: timestampInUnix,
         anonymousId: .message.anonymousId
       }

src/cdk/v2/destinations/yandex_metrica_offline_events/config.js

@@ -0,0 +1,5 @@
+const YANDEX_METRICA_OFFLINE_EVENTS = 'yandex_metrica_offline_events';
+
+module.exports = {
+  YANDEX_METRICA_OFFLINE_EVENTS,
+};

src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml

@@ -0,0 +1,36 @@
+bindings:
+  - name: EventType
+    path: ../../../../constants
+  - path: ../../bindings/jsontemplate
+    exportAll: true
+  - path: ./config
+  - name: removeUndefinedAndNullValues
+    path: ../../../../v0/util
+  - name: defaultRequestConfig
+    path: ../../../../v0/util
+  - path: ./utils
+
+steps:
+  - name: validateInput
+    template: |
+      let messageType = .message.type;
+      $.assert(messageType, "message Type is not present. Aborting message.");
+      $.assert(.message.type.toLowerCase() ==='identify', "Event type " + .message.type.toLowerCase() + " is not supported. Aborting message.");
+      $.assert(.message.traits || .message.properties, "Message traits/properties not present. Aborting message.");
+
+  - name: prepareData
+    template: |
+      let data = .message.traits
+      let identifierType = .message.context.externalId[0].identifierType;
+      let identifierValue = .message.context.externalId[0].id;
+      identifierValue = String(identifierValue);
+      data = $.setIdentifier(data, identifierType, identifierValue)
+      data = $.validateData(data)
+      data
+
+  - name: buildResponseForProcessTransformation
+    description: build response
+    template: |
+      const response = $.defaultRequestConfig();
+      response.body.JSON = $.outputs.prepareData
+      response

src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js

@@ -0,0 +1,51 @@
+/* eslint-disable no-param-reassign */
+const { InstrumentationError, isDefinedNotNullNotEmpty } = require('@rudderstack/integrations-lib');
+const moment = require('moment');
+
+const setIdentifier = (data, identifierType, identifierValue) => {
+  const updatedData = data;
+  if (identifierType === 'ClientId') {
+    updatedData.ClientId = identifierValue;
+  } else if (identifierType === 'YCLID') {
+    updatedData.Yclid = identifierValue;
+  } else if (identifierType === 'UserId') {
+    updatedData.UserId = identifierValue;
+  } else {
+    throw new InstrumentationError(
+      'Invalid identifier type passed in external Id. Valid types are ClientId, YCLID, UserId. Aborting!',
+    );
+  }
+  return updatedData;
+};
+
+function isUnixTimestamp(datetime) {
+  if (moment.unix(datetime).isValid()) {
+    return datetime;
+  }
+  const unixTimestamp = moment(datetime).unix();
+  if (moment.unix(unixTimestamp).isValid()) {
+    return unixTimestamp;
+  }
+  throw new InstrumentationError('Invalid timestamp. Aborting!');
+}
+
+const validateData = (data) => {
+  const { Price, DateTime } = data;
+  if (!isDefinedNotNullNotEmpty(data)) {
+    throw new InstrumentationError('No traits found in the payload. Aborting!');
+  }
+  if (Price && typeof Price !== 'number') {
+    throw new InstrumentationError('Price can only be a numerical value. Aborting!');
+  }
+  if (!isDefinedNotNullNotEmpty(DateTime)) {
+    throw new InstrumentationError('DateTime cannot be empty. Aborting!');
+  }
+  data.DateTime = String(isUnixTimestamp(DateTime));
+  return data;
+};
+
+module.exports = {
+  setIdentifier,
+  validateData,
+  isUnixTimestamp,
+};

src/v0/destinations/algolia/config.js

@@ -5,11 +5,13 @@ const CONFIG_CATEGORIES = {
   TRACK: { type: 'track', name: 'AlgoliaTrack' },
 };
 const EVENT_TYPES = ['click', 'view', 'conversion'];
+const ALLOWED_EVENT_SUBTYPES = ['addToCart', 'purchase'];
 const MAX_BATCH_SIZE = 1000;
 const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname);
 module.exports = {
   ENDPOINT,
   MAX_BATCH_SIZE,
   EVENT_TYPES,
   trackMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK.name],
+  ALLOWED_EVENT_SUBTYPES,
 };

src/v0/destinations/algolia/data/AlgoliaTrack.json

@@ -34,5 +34,15 @@
     "destKey": "positions",
     "sourceKeys": "properties.positions",
     "required": false
+  },
+  {
+    "destKey": "value",
+    "sourceKeys": "properties.value",
+    "required": false
+  },
+  {
+    "destKey": "currency",
+    "sourceKeys": "properties.currency",
+    "required": false
   }
 ]

src/v0/destinations/algolia/transform.js

@@ -16,7 +16,7 @@ const {
   handleRtTfSingleEventError,
 } = require('../../util/index');
 
-const { ENDPOINT, MAX_BATCH_SIZE, trackMapping } = require('./config');
+const { ENDPOINT, MAX_BATCH_SIZE, trackMapping, ALLOWED_EVENT_SUBTYPES } = require('./config');
 
 const {
   genericpayloadValidator,
@@ -38,6 +38,12 @@ const trackResponseBuilder = (message, { Config }) => {
   const eventMapping = eventTypeMapping(Config);
   payload.eventName = event;
   payload.eventType = getValueFromMessage(message, 'properties.eventType') || eventMapping[event];
+  if (
+    payload.eventType === 'conversion' &&
+    ALLOWED_EVENT_SUBTYPES.includes(getValueFromMessage(message, 'properties.eventSubtype'))
+  ) {
+    payload.eventSubtype = getValueFromMessage(message, 'properties.eventSubtype');
+  }
 
   if (!payload.eventType) {
     throw new InstrumentationError('eventType is mandatory for track call');
@@ -47,9 +53,13 @@ const trackResponseBuilder = (message, { Config }) => {
   if (event === 'product list viewed' || event === 'order completed') {
     const products = getValueFromMessage(message, 'properties.products');
     if (products) {
-      const { objectList, positionList } = createObjectArray(products, payload.eventType);
+      const { objectList, positionList, objectData } = createObjectArray(
+        products,
+        payload.eventType,
+      );
       const objLen = objectList.length;
       const posLen = positionList.length;
+      const objDataLen = objectData.length;
       if (objLen > 0) {
         payload.objectIDs = objectList;
         payload.objectIDs.splice(20);
@@ -58,10 +68,20 @@ const trackResponseBuilder = (message, { Config }) => {
         payload.positions = positionList;
         payload.positions.splice(20);
       }
+
+      if (objDataLen > 0) {
+        payload.objectData = objectData;
+      }
       // making size of object list and position list equal
       if (posLen > 0 && objLen > 0 && posLen !== objLen) {
         throw new InstrumentationError('length of objectId and position should be equal');
       }
+
+      if (objDataLen > 0 && objLen > 0 && objDataLen !== objLen) {
+        throw new InstrumentationError(
+          'length of objectId and properties.products array should be equal',
+        );
+      }
     }
   }
   // for all events either filter or objectID should be there

src/v0/destinations/algolia/util.js

@@ -1,4 +1,8 @@
-const { InstrumentationError } = require('@rudderstack/integrations-lib');
+const {
+  InstrumentationError,
+  isDefined,
+  removeUndefinedAndNullValues,
+} = require('@rudderstack/integrations-lib');
 const logger = require('../../../logger');
 const { EVENT_TYPES } = require('./config');
 
@@ -66,6 +70,8 @@ const genericpayloadValidator = (payload) => {
 const createObjectArray = (objects, eventType) => {
   const objectList = [];
   const positionList = [];
+  // eslint-disable-next-line sonarjs/no-unused-collection
+  const objectData = [];
   if (objects.length > 0) {
     objects.forEach((object, index) => {
       if (object.objectId) {
@@ -80,13 +86,22 @@ const createObjectArray = (objects, eventType) => {
           }
         } else {
           objectList.push(object.objectId);
+          if (eventType === 'conversion') {
+            const singleObjData = {
+              queryID: isDefined(object.queryID) ? `${object.queryID}` : null,
+              price: isDefined(object.price) ? `${object.price}` : null,
+              quantity: object.quantity,
+              discount: isDefined(object.discount) ? `${object.discount}` : null,
+            };
+            objectData.push(removeUndefinedAndNullValues(singleObjData));
+          }
         }
       } else {
         logger.error(`object at index ${index} dropped. objectId is required.`);
       }
     });
   }
-  return { objectList, positionList };
+  return { objectList, positionList, objectData };
 };
 
 const clickPayloadValidator = (payload) => {

src/v0/destinations/facebook_conversions/utils.js

@@ -134,7 +134,7 @@ const populateCustomDataBasedOnCategory = (customData, message, category, catego
       const { contentIds, contents } = populateContentsAndContentIDs([message.properties]);
       eventTypeCustomData = {
         ...eventTypeCustomData,
-        content_ids: contentIds,
+        content_ids: contentIds.length === 1 ? contentIds[0] : contentIds,
         contents,
         content_type: contentType,
         content_category: getContentCategory(contentCategory),

src/v0/destinations/hs/HSTransform-v2.js

@@ -110,11 +110,11 @@ const processIdentify = async (message, destination, propertyMap) => {
     GENERIC_TRUE_VALUES.includes(mappedToDestination.toString()) &&
     operation
   ) {
-    addExternalIdToHSTraits(message);
     if (!objectType) {
       throw new InstrumentationError('objectType not found');
     }
     if (operation === 'createObject') {
+      addExternalIdToHSTraits(message);
       endpoint = CRM_CREATE_UPDATE_ALL_OBJECTS.replace(':objectType', objectType);
     } else if (operation === 'updateObject' && getHsSearchId(message)) {
       const { hsSearchId } = getHsSearchId(message);