Merge branch 'develop' into enhance.freshsales

.gitignore

@@ -132,7 +132,7 @@ dist
 
 # Others
 **/.DS_Store
-
+.dccache
 
 .idea
 

CHANGELOG.md

@@ -2,6 +2,23 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.55.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.54.4...v1.55.0) (2024-02-05)
+
+
+### Features
+
+* add new stat for access token expired in fb custom audience ([#3043](https://github.com/rudderlabs/rudder-transformer/issues/3043)) ([1e6d540](https://github.com/rudderlabs/rudder-transformer/commit/1e6d540fafc61a84fbbaa63d4bc5b1edc17ec44e))
+* **intercom:** upgrade intercom version from 1.4 to 2.10 ([#2976](https://github.com/rudderlabs/rudder-transformer/issues/2976)) ([717639b](https://github.com/rudderlabs/rudder-transformer/commit/717639bcce605109b145eb4cc6836fe1589278fe))
+* onboard new destination rakuten ([#3046](https://github.com/rudderlabs/rudder-transformer/issues/3046)) ([c7c3110](https://github.com/rudderlabs/rudder-transformer/commit/c7c3110a4526e31bc296abb33f3246fa8eee049a))
+* trade desk real time conversions ([#3023](https://github.com/rudderlabs/rudder-transformer/issues/3023)) ([212d5f0](https://github.com/rudderlabs/rudder-transformer/commit/212d5f09d8addc618d4398029e62c9a18a9512cf))
+
+
+### Bug Fixes
+
+* adding map for marketo known values and javascript known values ([#3037](https://github.com/rudderlabs/rudder-transformer/issues/3037)) ([64ab555](https://github.com/rudderlabs/rudder-transformer/commit/64ab555d31b4c1c49863794444bd79b2b6a45927))
+* mixpanel timestamp in ms ([#3028](https://github.com/rudderlabs/rudder-transformer/issues/3028)) ([5ad55a2](https://github.com/rudderlabs/rudder-transformer/commit/5ad55a27c8b737fd96f65c68ba086769747c5360))
+* upgrade ua-parser-js from 1.0.35 to 1.0.37 ([9a4cdef](https://github.com/rudderlabs/rudder-transformer/commit/9a4cdef59ab1c2d9dc95eb8629a7009d8d633297))
+
 ### [1.54.4](https://github.com/rudderlabs/rudder-transformer/compare/v1.54.3...v1.54.4) (2024-01-31)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.54.4",
+  "version": "1.55.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -65,7 +65,7 @@
     "@ndhoule/extend": "^2.0.0",
     "@pyroscope/nodejs": "^0.2.6",
     "@rudderstack/integrations-lib": "^0.2.2",
-    "@rudderstack/workflow-engine": "^0.6.9",
+    "@rudderstack/workflow-engine": "^0.7.2",
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",
     "ajv-formats": "^2.1.1",

src/cdk/v2/destinations/intercom/procWorkflow.yaml

@@ -41,7 +41,7 @@ steps:
       version;
 
   - name: rEtlPayload
-    condition: .message.context.mappedToDestination === true
+    condition: .message.context.mappedToDestination
     template: |
       $.addExternalIdToTraits(.message);
       const payload = $.getFieldValueFromMessage(.message, "traits");

src/cdk/v2/destinations/rakuten/procWorkflow.yaml

@@ -23,7 +23,7 @@ steps:
     template: |
       const properties = $.constructProperties(.message);
       const lineItems = $.constructLineItems(.message.properties)
-      $.context.payload = {...properties,...lineItems,xml:1, mid:.destination.Config.mid}
+      $.context.payload = {...properties,...lineItems,xml:1,source:'rudderstack', mid:.destination.Config.mid}
       $.context.payload = $.removeUndefinedAndNullValues($.context.payload);
 
   - name: buildResponse

src/cdk/v2/destinations/the_trade_desk/utils.js

@@ -115,8 +115,10 @@ const prepareItemsPayload = (message) => {
   let items;
   const eventMapInfo = ECOMM_EVENT_MAP[event.toLowerCase()];
   if (eventMapInfo?.itemsArray) {
+    // if event is one of the supported ecommerce events and products array is present
     items = prepareItemsFromProducts(message);
   } else if (eventMapInfo) {
+    // if event is one of the supported ecommerce events and products array is not present
     items = prepareItemsFromProperties(message);
   }
   return items;
@@ -304,14 +306,17 @@ const enrichTrackPayload = (message, payload) => {
   if (eventsMapInfo && !eventsMapInfo.itemsArray) {
     const itemExclusionList = generateExclusionList(ITEM_CONFIGS);
     rawPayload = extractCustomFields(message, rawPayload, ['properties'], itemExclusionList);
-  } else {
-    // for custom events
+  } else if (eventsMapInfo) {
+    // for ecomm events with products array supports. e.g Order Completed event
     rawPayload = extractCustomFields(
       message,
       rawPayload,
       ['properties'],
       ['products', 'revenue', 'value'],
     );
+  } else {
+    // for custom events
+    rawPayload = extractCustomFields(message, rawPayload, ['properties'], ['value']);
   }
   return rawPayload;
 };

src/cdk/v2/destinations/the_trade_desk/utils.test.js

@@ -635,25 +635,35 @@ describe('enrichTrackPayload', () => {
         order_id: 'ord123',
         property1: 'value1',
         property2: 'value2',
+        revenue: 10,
+        value: 11,
+        products: [
+          {
+            product_id: 'prd123',
+            test: 'test',
+          },
+        ],
       },
     };
     const payload = {
       order_id: 'ord123',
+      value: 11,
     };
-    let expectedPayload = {
+    const expectedPayload = {
       order_id: 'ord123',
       property1: 'value1',
       property2: 'value2',
+      revenue: 10,
+      value: 11,
+      products: [
+        {
+          product_id: 'prd123',
+          test: 'test',
+        },
+      ],
     };
 
-    let result = enrichTrackPayload(message, payload);
+    const result = enrichTrackPayload(message, payload);
     expect(result).toEqual(expectedPayload);
-
-    expectedPayload = {
-      order_id: 'ord123',
-      property1: 'value1',
-      property2: 'value2',
-    };
-    expect(enrichTrackPayload(message, {})).toEqual(expectedPayload);
   });
 });

src/cdk/v2/handler.ts

@@ -50,16 +50,20 @@ export async function getWorkflowEngine(
 
 const workflowEnginePromiseMap = new Map();
 
-export function getCachedWorkflowEngine(
+export async function getCachedWorkflowEngine(
   destName: string,
   feature: string,
   bindings: Record<string, unknown> = {},
-): WorkflowEngine {
+): Promise<WorkflowEngine> {
   // Create a new instance of the engine for the destination if needed
   // TODO: Use cache to avoid long living engine objects
   workflowEnginePromiseMap[destName] = workflowEnginePromiseMap[destName] || new Map();
   if (!workflowEnginePromiseMap[destName][feature]) {
-    workflowEnginePromiseMap[destName][feature] = getWorkflowEngine(destName, feature, bindings);
+    workflowEnginePromiseMap[destName][feature] = await getWorkflowEngine(
+      destName,
+      feature,
+      bindings,
+    );
   }
   return workflowEnginePromiseMap[destName][feature];
 }
@@ -97,5 +101,8 @@ export function executeStep(
 ): Promise<StepOutput> {
   return workflowEngine
     .getStepExecutor(stepName)
-    .execute(input, Object.assign(workflowEngine.bindings, getEmptyExecutionBindings(), bindings));
+    .execute(
+      input,
+      Object.assign(workflowEngine.getBindings(), getEmptyExecutionBindings(), bindings),
+    );
 }

src/v0/destinations/am/transform.js

@@ -22,13 +22,13 @@ const {
   getFieldValueFromMessage,
   getValueFromMessage,
   deleteObjectProperty,
-  getErrorRespEvents,
   removeUndefinedAndNullValues,
   isDefinedAndNotNull,
   isAppleFamily,
   isDefinedAndNotNullAndNotEmpty,
   simpleProcessRouterDest,
   isValidInteger,
+  handleRtTfSingleEventError,
 } = require('../../util');
 const {
   BASE_URL,
@@ -40,7 +40,6 @@ const {
   AMBatchSizeLimit,
   AMBatchEventLimit,
 } = require('./config');
-const tags = require('../../util/tags');
 
 const AMUtils = require('./utils');
 
@@ -615,16 +614,16 @@ const processSingleMessage = (message, destination) => {
     case EventType.PAGE:
       if (useUserDefinedPageEventName) {
         const getMessagePath = userProvidedPageEventString
-          .substring(
+          ?.substring(
             userProvidedPageEventString.indexOf('{') + 2,
             userProvidedPageEventString.indexOf('}'),
           )
           .trim();
         evType =
-          userProvidedPageEventString.trim() === ''
+          userProvidedPageEventString?.trim() === ''
             ? name
             : userProvidedPageEventString
-                .trim()
+                ?.trim()
                 .replaceAll(/{{([^{}]+)}}/g, get(message, getMessagePath));
       } else {
         evType = `Viewed ${name || get(message, CATEGORY_KEY) || ''} Page`;
@@ -702,6 +701,7 @@ const processSingleMessage = (message, destination) => {
       logger.debug('could not determine type');
       throw new InstrumentationError('message type not supported');
   }
+  AMUtils.validateEventType(evType);
   return responseBuilderSimple(
     groupInfo,
     payloadObjectName,
@@ -904,16 +904,10 @@ const batch = (destEvents) => {
     // this case shold not happen and should be filtered already
     // by the first pass of single event transformation
     if (messageEvent && !userId && !deviceId) {
-      const errorResponse = getErrorRespEvents(
-        metadata,
-        400,
+      const MissingUserIdDeviceIdError = new InstrumentationError(
         'Both userId and deviceId cannot be undefined',
-        {
-          [tags.TAG_NAMES.ERROR_CATEGORY]: tags.ERROR_CATEGORIES.DATA_VALIDATION,
-          [tags.TAG_NAMES.ERROR_TYPE]: tags.ERROR_TYPES.INSTRUMENTATION,
-        },
       );
-      respList.push(errorResponse);
+      respList.push(handleRtTfSingleEventError(ev, MissingUserIdDeviceIdError, {}));
       return;
     }
     /* check if not a JSON body or (userId length < 5 && batchEventsWithUserIdLengthLowerThanFive is false) or

src/v0/destinations/am/util.test.js

@@ -1,4 +1,4 @@
-const { getUnsetObj } = require('./utils');
+const { getUnsetObj, validateEventType } = require('./utils');
 
 describe('getUnsetObj', () => {
   it("should return undefined when 'message.integrations.Amplitude.fieldsToUnset' is not array", () => {
@@ -64,3 +64,34 @@ describe('getUnsetObj', () => {
     expect(result).toBeUndefined();
   });
 });
+
+
+describe('validateEventType', () => {
+
+  it('should validate event type when it is valid with only page name given', () => {
+    expect(() => {
+      validateEventType('Home Page');
+    }).not.toThrow();
+  });
+
+  it('should throw an error when event type is null', () => {
+    expect(() => {
+      validateEventType(null);
+    }).toThrow('Event type is missing. Please send it under `event.type`. For page/screen events, send it under `event.name`');
+  });
+
+  it('should throw an error when event type is undefined', () => {
+    expect(() => {
+      validateEventType(undefined);
+    }).toThrow('Event type is missing. Please send it under `event.type`. For page/screen events, send it under `event.name`');
+  });
+
+  // Function receives an empty string as event type
+  it('should throw an error when event type is an empty string', () => {
+    expect(() => {
+      validateEventType('');
+    }).toThrow('Event type is missing. Please send it under `event.type`. For page/screen events, send it under `event.name`');
+  });
+
+});
+

src/v0/destinations/am/utils.js

@@ -10,6 +10,7 @@
 // populate these dest keys
 const get = require('get-value');
 const uaParser = require('@amplitude/ua-parser-js');
+const { InstrumentationError } = require('@rudderstack/integrations-lib');
 const logger = require('../../../logger');
 const { isDefinedAndNotNull } = require('../../util');
 
@@ -108,6 +109,19 @@ const getUnsetObj = (message) => {
 
   return unsetObject;
 };
+
+/**
+ * Check for evType as in some cases, like when the page name is absent, 
+ * either the template depends only on the event.name or there is no template provided by user
+ * @param {*} evType 
+ */
+const validateEventType = (evType) => {
+    if (!isDefinedAndNotNull(evType) || (typeof evType === "string" && evType.length ===0)) {
+      throw new InstrumentationError(
+        'Event type is missing. Please send it under `event.type`. For page/screen events, send it under `event.name`',
+      );
+    }
+};
 module.exports = {
   getOSName,
   getOSVersion,
@@ -117,4 +131,5 @@ module.exports = {
   getBrand,
   getEventId,
   getUnsetObj,
+  validateEventType
 };

src/v0/destinations/ga4/networkHandler.js

@@ -30,9 +30,9 @@ const responseHandler = (destinationResponse, dest) => {
       const { description, validationCode, fieldPath } = response.validationMessages[0];
       throw new NetworkError(
         `Validation Server Response Handler:: Validation Error for ${dest} of field path :${fieldPath} | ${validationCode}-${description}`,
-        status,
+        400,
         {
-          [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status),
+          [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(400),
         },
         response?.validationMessages[0]?.description,
       );