Merge branch 'develop' into fix.handle-circular-struct-response

CHANGELOG.md

@@ -2,6 +2,53 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.81.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.79.1...v1.81.0) (2024-10-04)
+
+
+### Features
+
+* add unity source support in Singular ([#3634](https://github.com/rudderlabs/rudder-transformer/issues/3634)) ([12996d7](https://github.com/rudderlabs/rudder-transformer/commit/12996d7a7ce23de7c150c1c1e012d4dda8668977))
+* onboard shopify to v1 ([#3665](https://github.com/rudderlabs/rudder-transformer/issues/3665)) ([d40e772](https://github.com/rudderlabs/rudder-transformer/commit/d40e772f1a3741c1c4e9ab2365ed464b3988812e))
+
+
+### Bug Fixes
+
+* add correct validation for purchase events ([#3766](https://github.com/rudderlabs/rudder-transformer/issues/3766)) ([9cc72f2](https://github.com/rudderlabs/rudder-transformer/commit/9cc72f2288f99ee394977ffeb209faaae657f6d2))
+* braze include fields_to_export to lookup users ([#3761](https://github.com/rudderlabs/rudder-transformer/issues/3761)) ([173b989](https://github.com/rudderlabs/rudder-transformer/commit/173b9895fb2a0bed615f6e3a9c670abe42d5754f))
+* correct typo for order fulfillment event, add test ([#3764](https://github.com/rudderlabs/rudder-transformer/issues/3764)) ([6f92bd3](https://github.com/rudderlabs/rudder-transformer/commit/6f92bd31b60caaa07d18bb86ce5939cd7cc9a416))
+* fixing lytics user_id and anonymousId mapping ([#3745](https://github.com/rudderlabs/rudder-transformer/issues/3745)) ([45b1067](https://github.com/rudderlabs/rudder-transformer/commit/45b1067d81f3883e19d35634ffec52434fef452f))
+* npm start command to include exec ([9f5140b](https://github.com/rudderlabs/rudder-transformer/commit/9f5140b194384295c0a56147fed16273b2b7805b))
+* payment info entered event in facebook_conversions ([#3762](https://github.com/rudderlabs/rudder-transformer/issues/3762)) ([7fa7c8d](https://github.com/rudderlabs/rudder-transformer/commit/7fa7c8d3a4f6aefb580cf0de2e64e2f8aef5b5ce))
+* posthog alias mapping swap ([#3765](https://github.com/rudderlabs/rudder-transformer/issues/3765)) ([b6240d0](https://github.com/rudderlabs/rudder-transformer/commit/b6240d06a9d1f7f3bc8f245807f72a72ab40f170)), closes [#3507](https://github.com/rudderlabs/rudder-transformer/issues/3507)
+
+## [1.80.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.79.1...v1.80.0) (2024-09-30)
+
+
+### Features
+
+* add unity source support in Singular ([#3634](https://github.com/rudderlabs/rudder-transformer/issues/3634)) ([12996d7](https://github.com/rudderlabs/rudder-transformer/commit/12996d7a7ce23de7c150c1c1e012d4dda8668977))
+* onboard shopify to v1 ([#3665](https://github.com/rudderlabs/rudder-transformer/issues/3665)) ([d40e772](https://github.com/rudderlabs/rudder-transformer/commit/d40e772f1a3741c1c4e9ab2365ed464b3988812e))
+
+
+### Bug Fixes
+
+* add correct validation for purchase events ([#3766](https://github.com/rudderlabs/rudder-transformer/issues/3766)) ([9cc72f2](https://github.com/rudderlabs/rudder-transformer/commit/9cc72f2288f99ee394977ffeb209faaae657f6d2))
+* braze include fields_to_export to lookup users ([#3761](https://github.com/rudderlabs/rudder-transformer/issues/3761)) ([173b989](https://github.com/rudderlabs/rudder-transformer/commit/173b9895fb2a0bed615f6e3a9c670abe42d5754f))
+* correct typo for order fulfillment event, add test ([#3764](https://github.com/rudderlabs/rudder-transformer/issues/3764)) ([6f92bd3](https://github.com/rudderlabs/rudder-transformer/commit/6f92bd31b60caaa07d18bb86ce5939cd7cc9a416))
+* fixing lytics user_id and anonymousId mapping ([#3745](https://github.com/rudderlabs/rudder-transformer/issues/3745)) ([45b1067](https://github.com/rudderlabs/rudder-transformer/commit/45b1067d81f3883e19d35634ffec52434fef452f))
+* payment info entered event in facebook_conversions ([#3762](https://github.com/rudderlabs/rudder-transformer/issues/3762)) ([7fa7c8d](https://github.com/rudderlabs/rudder-transformer/commit/7fa7c8d3a4f6aefb580cf0de2e64e2f8aef5b5ce))
+* posthog alias mapping swap ([#3765](https://github.com/rudderlabs/rudder-transformer/issues/3765)) ([b6240d0](https://github.com/rudderlabs/rudder-transformer/commit/b6240d06a9d1f7f3bc8f245807f72a72ab40f170)), closes [#3507](https://github.com/rudderlabs/rudder-transformer/issues/3507)
+
+### [1.79.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.79.0...v1.79.1) (2024-09-24)
+
+
+### Bug Fixes
+
+* allow users context traits and underscore divide numbers configuration ([#3752](https://github.com/rudderlabs/rudder-transformer/issues/3752)) ([386d2ab](https://github.com/rudderlabs/rudder-transformer/commit/386d2ab88c0fe72dc47ba119be08ad1c0cd6d51b))
+* populate users fields for sentAt, timestamp and originalTimestamp ([#3753](https://github.com/rudderlabs/rudder-transformer/issues/3753)) ([f50effe](https://github.com/rudderlabs/rudder-transformer/commit/f50effeeabdb888f82451c225a80971dbe6532b6))
+* prefer event check vs config check for vdm ([#3754](https://github.com/rudderlabs/rudder-transformer/issues/3754)) ([b2c1a18](https://github.com/rudderlabs/rudder-transformer/commit/b2c1a1893dfb957ac7a24c000b33cd254ef54b6c))
+* support different lookup fields and custom_attributes for rETL events  ([#3751](https://github.com/rudderlabs/rudder-transformer/issues/3751)) ([10d914e](https://github.com/rudderlabs/rudder-transformer/commit/10d914e25203bd6ae95801c2a98c17690bd2d6ef))
+
 ## [1.79.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.78.0...v1.79.0) (2024-09-20)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.79.0",
+  "version": "1.81.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -23,7 +23,7 @@
     "lint:fix:json": "eslint --ext .json --fix .",
     "lint": "npm run format && npm run lint:fix",
     "check:merge": "npm run verify || exit 1; codecov",
-    "start": "cd dist;node ./src/index.js;cd ..",
+    "start": "cd dist;exec node ./src/index.js;cd ..",
     "build:start": "npm run build && npm run start",
     "build:ci": "tsc -p tsconfig.json",
     "build:swagger": "npm run build && npm run setup:swagger",
@@ -71,6 +71,7 @@
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",
     "ajv-formats": "^2.1.1",
+    "amazon-dsp-formatter": "^1.0.2",
     "axios": "^1.7.3",
     "btoa": "^1.2.1",
     "component-each": "^0.2.6",

src/cdk/v2/destinations/bluecore/utils.js

@@ -46,12 +46,12 @@ const verifyPayload = (payload, message) => {
       }
       break;
     case 'purchase':
-      if (!payload?.properties?.order_id) {
+      if (!isDefinedAndNotNull(payload?.properties?.order_id)) {
         throw new InstrumentationError(
           '[Bluecore] property:: order_id is required for purchase event',
         );
       }
-      if (!payload?.properties?.total) {
+      if (!isDefinedAndNotNull(payload?.properties?.total)) {
         throw new InstrumentationError(
           '[Bluecore] property:: total is required for purchase event',
         );

src/cdk/v2/destinations/intercom/procWorkflow.yaml

@@ -77,8 +77,8 @@ steps:
     template: |
       const payload = .message.context.mappedToDestination ? $.outputs.rEtlPayload : $.outputs.identifyTransformationForLatestVersion;
       payload.name = $.getName(.message);
-      payload.custom_attributes = .message.context.traits || {};
-      payload.custom_attributes = $.filterCustomAttributes(payload, "user", .destination);
+      payload.custom_attributes = (.message.context.mappedToDestination ? .message.traits.custom_attributes : .message.context.traits) || {};
+      payload.custom_attributes = $.filterCustomAttributes(payload, "user", .destination, .message);
       payload.external_id = !payload.external_id && .destination.Config.sendAnonymousId && .message.anonymousId ? .message.anonymousId : payload.external_id;
       $.context.payload = payload;
       $.assert($.context.payload.external_id || $.context.payload.email, "Either email or userId is required for Identify call");
@@ -114,7 +114,7 @@ steps:
        update_last_request_at: typeof .destination.Config.updateLastRequestAt === 'boolean' ? .destination.Config.updateLastRequestAt : true
       }
       payload.companies = $.getCompaniesList(payload);
-      payload.custom_attributes = !.message.context.mappedToDestination ? $.filterCustomAttributes(payload, "user", .destination);
+      payload.custom_attributes = !.message.context.mappedToDestination ? $.filterCustomAttributes(payload, "user", .destination,.message);
       payload.user_id = !payload.user_id && .destination.Config.sendAnonymousId && .message.anonymousId ? .message.anonymousId : payload.user_id;
       $.context.payload = payload;
       $.assert($.context.payload.user_id || $.context.payload.email, "Either of `email` or `userId` is required for Identify call");
@@ -175,7 +175,7 @@ steps:
           $.assert(.message.groupId, "groupId is required for group call");
           const payload = .message.context.mappedToDestination ? $.outputs.rEtlPayload : $.outputs.groupTransformation;
           payload.custom_attributes = .message.traits || {};
-          payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination);
+          payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination,.message);
           $.context.payload = payload;
       - name: whenSearchContactFound
         condition: $.isDefinedAndNotNull($.outputs.searchContact)
@@ -214,7 +214,7 @@ steps:
         ...payload,
         custom_attributes : $.getFieldValueFromMessage(.message, "traits") || {}
       }
-      payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination);
+      payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination, .message);
       response.body.JSON = $.removeUndefinedAndNullValues(payload);
       response.endpoint = $.getBaseEndpoint(.destination) + "/" + "companies";
       response.headers = $.getHeaders(.destination, $.outputs.apiVersion);

src/cdk/v2/destinations/intercom/utils.js

@@ -233,20 +233,26 @@ const attachUserAndCompany = (message, Config) => {
  * @param {*} type
  * @returns
  */
-const filterCustomAttributes = (payload, type, destination) => {
+const filterCustomAttributes = (payload, type, destination, message) => {
   let ReservedAttributesList;
   let { apiVersion } = destination.Config;
   apiVersion = isDefinedAndNotNull(apiVersion) ? apiVersion : 'v2';
+  // we are discarding the lookup field from custom attributes
+  const lookupField = getLookUpField(message);
   if (type === 'user') {
-    ReservedAttributesList =
-      apiVersion === 'v1'
+    ReservedAttributesList = [
+      ...(apiVersion === 'v1'
         ? ReservedAttributes.v1UserAttributes
-        : ReservedAttributes.v2UserAttributes;
+        : ReservedAttributes.v2UserAttributes),
+      lookupField,
+    ];
   } else {
-    ReservedAttributesList =
-      apiVersion === 'v1'
+    ReservedAttributesList = [
+      ...(apiVersion === 'v1'
         ? ReservedAttributes.v1CompanyAttributes
-        : ReservedAttributes.v2CompanyAttributes;
+        : ReservedAttributes.v2CompanyAttributes),
+      lookupField !== 'email' && lookupField,
+    ];
   }
   let customAttributes = { ...get(payload, 'custom_attributes') };
   if (customAttributes) {
@@ -270,7 +276,10 @@ const filterCustomAttributes = (payload, type, destination) => {
  */
 const searchContact = async (message, destination, metadata) => {
   const lookupField = getLookUpField(message);
-  const lookupFieldValue = getFieldValueFromMessage(message, lookupField);
+  let lookupFieldValue = getFieldValueFromMessage(message, lookupField);
+  if (!lookupFieldValue) {
+    lookupFieldValue = message?.context?.traits?.[lookupField];
+  }
   const data = JSON.stringify({
     query: {
       operator: 'AND',

src/cdk/v2/destinations/lytics/config.ts

@@ -0,0 +1,9 @@
+import { getMappingConfig } from '../../../../v0/util';
+
+const CONFIG_CATEGORIES = {
+  CUSTOMER_PROPERTIES_CONFIG: { name: 'LYTICSIdentifyConfig' },
+};
+
+const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname);
+export const CUSTOMER_PROPERTIES_CONFIG =
+  MAPPING_CONFIG[CONFIG_CATEGORIES.CUSTOMER_PROPERTIES_CONFIG.name];

src/cdk/v2/destinations/lytics/data/LYTICSIdentifyConfig.json

@@ -0,0 +1,25 @@
+[
+  {
+    "destKey": "user_id",
+    "sourceKeys": "userIdOnly",
+    "sourceFromGenericMap": true,
+    "required": false
+  },
+  {
+    "destKey": "anonymous_id",
+    "sourceKeys": "anonymousId",
+    "required": false
+  },
+  {
+    "destKey": "first_name",
+    "sourceKeys": "firstName",
+    "sourceFromGenericMap": true,
+    "required": false
+  },
+  {
+    "destKey": "last_name",
+    "sourceKeys": "lastName",
+    "sourceFromGenericMap": true,
+    "required": false
+  }
+]

src/cdk/v2/destinations/lytics/procWorkflow.yaml

@@ -5,9 +5,12 @@ bindings:
     path: ../../../../v0/util
   - name: removeUndefinedAndNullValues
     path: ../../../../v0/util
+  - name: constructPayload
+    path: ../../../../v0/util
   - path: ../../bindings/jsontemplate
   - name: defaultRequestConfig
     path: ../../../../v0/util
+  - path: ./config
 
 steps:
   - name: validateInput
@@ -24,20 +27,19 @@ steps:
     condition: $.context.messageType === {{$.EventType.IDENTIFY}}
     template: |
       const flattenTraits = $.flattenJson(.message.traits ?? .message.context.traits);
-      $.context.payload = .message.({
+      const payload = $.constructPayload(.message, $.CUSTOMER_PROPERTIES_CONFIG);
+      $.context.payload = {
         ...flattenTraits,
-        first_name: {{{{$.getGenericPaths("firstName")}}}},
-        last_name: {{{{$.getGenericPaths("lastName")}}}},
-        user_id: {{{{$.getGenericPaths("userId")}}}}
-      })
+        ...payload,
+      }
     else:
       name: payloadForOthers
       template: |
         const flattenProperties = $.flattenJson(.message.properties);
+        const customerPropertiesInfo = $.constructPayload(.message, $.CUSTOMER_PROPERTIES_CONFIG);
         $.context.payload = .message.({
           ...flattenProperties,
-          first_name: .properties.firstName ?? .properties.firstname,
-          last_name: .properties.lastName ?? .properties.lastname
+          ...customerPropertiesInfo
         })
   - name: trackPayload
     condition: $.context.messageType === {{$.EventType.TRACK}}

src/cdk/v2/destinations/rakuten/utils.js

@@ -17,6 +17,18 @@ const constructProperties = (message) => {
   return payload;
 };
 
+/**
+ * Calculates the amount for a single product
+ * @param {Object} product
+ * @returns {number}
+ */
+const calculateProductAmount = (product) => {
+  if (!product?.amount && !product?.price) {
+    throw new InstrumentationError('Either amount or price is required for every product');
+  }
+  return Math.round(product.amount * 100 || (product.quantity || 1) * 100 * product.price);
+};
+
 /**
  * This fucntion build the item level list
  * @param {*} properties
@@ -52,14 +64,8 @@ const constructLineItems = (properties) => {
   });
 
   // Map 'amountList' by evaluating 'amount' or deriving it from 'price' and 'quantity'
-  const amountList = products.map((product) => {
-    if (!product?.amount && !product?.price) {
-      throw new InstrumentationError('Either amount or price is required for every product');
-    }
-    return product.amount * 100 || (product.quantity || 1) * 100 * product.price;
-  });
-  productList.amtlist = amountList.join('|');
+  productList.amtlist = products.map(calculateProductAmount).join('|');
   return productList;
 };
 
-module.exports = { constructProperties, constructLineItems };
+module.exports = { constructProperties, constructLineItems, calculateProductAmount };

src/cdk/v2/destinations/rakuten/utils.test.js

@@ -1,4 +1,5 @@
-const { constructLineItems } = require('./utils');
+const { InstrumentationError } = require('@rudderstack/integrations-lib');
+const { constructLineItems, calculateProductAmount } = require('./utils');
 describe('constructLineItems', () => {
   it('should return a non-empty object when given a valid properties object with at least one product', () => {
     const properties = {
@@ -115,3 +116,62 @@ describe('constructLineItems', () => {
     );
   });
 });
+
+describe('calculateProductAmount', () => {
+  // Calculates product amount correctly when amount is defined
+  it('should return the correct product amount when amount is defined', () => {
+    const product = { amount: 5 };
+    const result = calculateProductAmount(product);
+    expect(result).toBe(500);
+  });
+
+  // Throws error when both amount and price are undefined or null
+  it('should throw an error when both amount and price are undefined or null', () => {
+    const product = {};
+    expect(() => calculateProductAmount(product)).toThrow(InstrumentationError);
+  });
+
+  // Calculates product amount correctly when price and quantity are defined
+  it('should calculate product amount correctly when price and quantity are defined', () => {
+    const product = { amount: 10, price: 5, quantity: 2 };
+    const result = calculateProductAmount(product);
+    expect(result).toEqual(1000);
+  });
+
+  // Returns correct value when only price is defined and quantity defaults to 1
+  it('should return correct value when only price is defined and quantity defaults to 1', () => {
+    const product = { price: 20 };
+    const result = calculateProductAmount(product);
+    expect(result).toEqual(2000);
+  });
+
+  // Handles cases where amount is a floating-point number
+  it('should handle cases where amount is a floating-point number', () => {
+    const product = { amount: 5.5, price: 10, quantity: 2 };
+    const result = calculateProductAmount(product);
+    expect(result).toEqual(550);
+  });
+
+  it('should handle cases where amount is a floating-point number', () => {
+    const product = { amount: 5.1, price: 10, quantity: 2 };
+    const result = calculateProductAmount(product);
+    expect(result).toEqual(510);
+  });
+
+  it('should handle cases where amount is a floating-point number', () => {
+    const product = { amount: 5.19, price: 10, quantity: 2 };
+    const result = calculateProductAmount(product);
+    expect(result).toEqual(519);
+  });
+
+  it('should handle cases where amount is a floating-point number', () => {
+    const product = { amount: 5.199, price: 10, quantity: 2 };
+    const result = calculateProductAmount(product);
+    expect(result).toEqual(520);
+  });
+  it('should handle cases where amount is a floating-point number', () => {
+    const product = { amount: 5.479, price: 10, quantity: 2 };
+    const result = calculateProductAmount(product);
+    expect(result).toEqual(548);
+  });
+});