Merge branch 'develop' into fix.capiHashingStandardisation

.github/workflows/build-push-docker-image.yml

@@ -38,6 +38,7 @@ jobs:
       - name: Checkout
         uses: actions/[email protected]
         with:
+          ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 1
 
       - name: Setup Docker Buildx
@@ -88,6 +89,7 @@ jobs:
       - name: Checkout
         uses: actions/[email protected]
         with:
+          ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 1
 
       - name: Setup Docker Buildx

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.66.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.66.0...v1.66.1) (2024-05-20)
+
+
+### Bug Fixes
+
+* add validation for null/undefined traits in slack ([#3382](https://github.com/rudderlabs/rudder-transformer/issues/3382)) ([755073c](https://github.com/rudderlabs/rudder-transformer/commit/755073c4341a454785050d835021d9f17e0b9d3f))
+* gaoc store sales batching transform contract ([#3384](https://github.com/rudderlabs/rudder-transformer/issues/3384)) ([e7678cb](https://github.com/rudderlabs/rudder-transformer/commit/e7678cbdae4c06449ea9352ce3db390d2a29da14))
+* move af_currency outside properties in eventValue ([#3316](https://github.com/rudderlabs/rudder-transformer/issues/3316)) ([71c3d46](https://github.com/rudderlabs/rudder-transformer/commit/71c3d46236fff9209625cfb0737c21db2d275345))
+* remove default traits from ortto ([#3389](https://github.com/rudderlabs/rudder-transformer/issues/3389)) ([fbb0811](https://github.com/rudderlabs/rudder-transformer/commit/fbb0811aa0e417b0cffcea4ecc103979afccfe74))
+* update validation of event duration ([#3376](https://github.com/rudderlabs/rudder-transformer/issues/3376)) ([3ad7850](https://github.com/rudderlabs/rudder-transformer/commit/3ad78506446915ada8bdc5f5594dc2710e6b0646))
+
 ## [1.66.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.65.1...v1.66.0) (2024-05-13)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.66.0",
+  "version": "1.66.1",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/cdk/v2/destinations/ortto/procWorkflow.yaml

@@ -48,9 +48,9 @@ steps:
           "str::ei": {{{{$.getGenericPaths("userId")}}}},
           "str::language": .context.traits.language || .context.locale,
           "phn::phone": phone ? {"n": phone},
-          "bol::gdpr": .context.traits.gdpr ?? true,
-          "bol::p": .context.traits.emailConsent || false,
-          "bol::sp": .context.traits.smsConsent || false,
+          "bol::gdpr": .context.traits.gdpr,
+          "bol::p": .context.traits.emailConsent,
+          "bol::sp": .context.traits.smsConsent,
           },
         "location": {"source_ip": .context.ip}
       });

src/v0/destinations/af/transform.js

@@ -113,13 +113,9 @@ function getEventValueForUnIdentifiedTrackEvent(message) {
   return { eventValue };
 }
 
-function getEventValueMapFromMappingJson(
-  message,
-  mappingJson,
-  isMultiSupport,
-  addPropertiesAtRoot,
-) {
+function getEventValueMapFromMappingJson(message, mappingJson, isMultiSupport, config) {
   let eventValue = {};
+  const { addPropertiesAtRoot, afCurrencyAtRoot } = config;
 
   if (addPropertiesAtRoot) {
     eventValue = message.properties;
@@ -152,6 +148,9 @@ function getEventValueMapFromMappingJson(
       af_price: prices,
     };
   }
+  if (afCurrencyAtRoot) {
+    eventValue.af_currency = message.properties.currency;
+  }
   eventValue = removeUndefinedValues(eventValue);
   if (Object.keys(eventValue).length > 0) {
     eventValue = JSON.stringify(eventValue);
@@ -171,7 +170,7 @@ function processNonTrackEvents(message, eventName) {
   return payload;
 }
 
-function processEventTypeTrack(message, addPropertiesAtRoot) {
+function processEventTypeTrack(message, config) {
   let isMultiSupport = true;
   const evType = message.event && message.event.toLowerCase();
   let category = ConfigCategory.DEFAULT;
@@ -195,7 +194,7 @@ function processEventTypeTrack(message, addPropertiesAtRoot) {
     message,
     mappingConfig[category.name],
     isMultiSupport,
-    addPropertiesAtRoot,
+    config,
   );
   payload.eventName = message.event;
   payload.eventCurrency = message?.properties?.currency;
@@ -208,7 +207,7 @@ function processSingleMessage(message, destination) {
   let payload;
   switch (messageType) {
     case EventType.TRACK: {
-      payload = processEventTypeTrack(message, destination.Config.addPropertiesAtRoot);
+      payload = processEventTypeTrack(message, destination.Config);
       break;
     }
     case EventType.SCREEN: {

src/v0/destinations/facebook_conversions/transform.js

@@ -1,6 +1,5 @@
 /* eslint-disable no-param-reassign */
 const get = require('get-value');
-const moment = require('moment');
 const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib');
 const {
   CONFIG_CATEGORIES,
@@ -33,6 +32,7 @@ const {
   fetchUserData,
   formingFinalResponse,
 } = require('../../util/facebookUtils');
+const { verifyEventDuration } = require('../facebook_pixel/utils');
 
 const responseBuilderSimple = (message, category, destination) => {
   const { Config, ID } = destination;
@@ -121,19 +121,7 @@ const processEvent = (message, destination) => {
   }
 
   const timeStamp = getFieldValueFromMessage(message, 'timestamp');
-  if (timeStamp) {
-    const start = moment.unix(moment(timeStamp).format('X'));
-    const current = moment.unix(moment().format('X'));
-    // calculates past event in days
-    const deltaDay = Math.ceil(moment.duration(current.diff(start)).asDays());
-    // calculates future event in minutes
-    const deltaMin = Math.ceil(moment.duration(start.diff(current)).asMinutes());
-    if (deltaDay > 7 || deltaMin > 1) {
-      throw new InstrumentationError(
-        'Events must be sent within seven days of their occurrence or up to one minute in the future.',
-      );
-    }
-  }
+  verifyEventDuration(message, destination, timeStamp);
 
   const { datasetId, accessToken } = destination.Config;
   if (!datasetId) {

src/v0/destinations/facebook_pixel/transform.js

@@ -1,8 +1,6 @@
 /* eslint-disable no-param-reassign */
 const get = require('get-value');
-const moment = require('moment');
 const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib');
-const stats = require('../../../util/stats');
 const {
   VERSION,
   CONFIG_CATEGORIES,
@@ -31,6 +29,7 @@ const {
   handleOrder,
   populateCustomDataBasedOnCategory,
   getCategoryFromEvent,
+  verifyEventDuration,
 } = require('./utils');
 
 const {
@@ -170,23 +169,7 @@ const processEvent = (message, destination) => {
   }
 
   const timeStamp = message.timestamp || message.originalTimestamp;
-  if (timeStamp) {
-    const start = moment.unix(moment(timeStamp).format('X'));
-    const current = moment.unix(moment().format('X'));
-    // calculates past event in days
-    const deltaDay = Math.ceil(moment.duration(current.diff(start)).asDays());
-    // calculates future event in minutes
-    const deltaMin = Math.ceil(moment.duration(start.diff(current)).asMinutes());
-    if (deltaDay > 7 || deltaMin > 1) {
-      // TODO: Remove after testing in mirror transformer
-      stats.increment('fb_pixel_timestamp_error', {
-        destinationId: destination.ID,
-      });
-      throw new InstrumentationError(
-        'Events must be sent within seven days of their occurrence or up to one minute in the future.',
-      );
-    }
-  }
+  verifyEventDuration(message, destination, timeStamp);
 
   let eventsToEvents;
   if (Array.isArray(destination.Config.eventsToEvents)) {

src/v0/destinations/facebook_pixel/utils.js

@@ -1,4 +1,6 @@
 const { InstrumentationError } = require('@rudderstack/integrations-lib');
+const get = require('get-value');
+const moment = require('moment');
 const { isObject } = require('../../util');
 const {
   ACTION_SOURCES_VALUES,
@@ -339,6 +341,29 @@ const getCategoryFromEvent = (eventName) => {
   return category;
 };
 
+const verifyEventDuration = (message, destination, timeStamp) => {
+  const actionSource =
+    get(message, 'traits.action_source') ||
+    get(message, 'context.traits.action_source') ||
+    get(message, 'properties.action_source');
+
+  const start = moment.unix(moment(timeStamp).format('X'));
+  const current = moment.unix(moment().format('X'));
+  // calculates past event in days
+  const deltaDay = Math.ceil(moment.duration(current.diff(start)).asDays());
+  // calculates future event in minutes
+  const deltaMin = Math.ceil(moment.duration(start.diff(current)).asMinutes());
+  let defaultSupportedDelta = 7;
+  if (actionSource === 'physical_store') {
+    defaultSupportedDelta = 62;
+  }
+  if (deltaDay > defaultSupportedDelta || deltaMin > 1) {
+    throw new InstrumentationError(
+      `Events must be sent within ${defaultSupportedDelta} days of their occurrence or up to one minute in the future.`,
+    );
+  }
+};
+
 module.exports = {
   formatRevenue,
   getActionSource,
@@ -348,4 +373,5 @@ module.exports = {
   handleOrder,
   populateCustomDataBasedOnCategory,
   getCategoryFromEvent,
+  verifyEventDuration,
 };

src/v0/destinations/facebook_pixel/utils.test.js

@@ -1,7 +1,13 @@
 const { InstrumentationError } = require('@rudderstack/integrations-lib');
-const { getActionSource, formatRevenue, getCategoryFromEvent } = require('./utils');
+const {
+  getActionSource,
+  formatRevenue,
+  getCategoryFromEvent,
+  verifyEventDuration,
+} = require('./utils');
 const { CONFIG_CATEGORIES, OTHER_STANDARD_EVENTS } = require('./config');
 
+Date.now = jest.fn(() => new Date('2022-01-20T00:00:00Z'));
 describe('Test Facebook Pixel Utils', () => {
   describe('getActionSource', () => {
     // Returns 'other' if payload.action_source is not defined and channel is neither 'web' nor 'mobile'
@@ -151,4 +157,75 @@ describe('Test Facebook Pixel Utils', () => {
       expect(result).toEqual(CONFIG_CATEGORIES.SIMPLE_TRACK);
     });
   });
+
+  describe('verifyEventDuration', () => {
+    it('should not throw an InstrumentationError when event duration is less than 8 days after the event occurred', () => {
+      const message = {
+        traits: {
+          action_source: 'some_action_source',
+        },
+        context: {
+          traits: {
+            action_source: 'some_action_source',
+          },
+        },
+        properties: {
+          action_source: 'some_action_source',
+        },
+      };
+      const destination = {
+        ID: 'some_destination_id',
+      };
+      const timeStamp = '2022-01-20T00:00:00Z';
+      expect(() => {
+        verifyEventDuration(message, destination, timeStamp);
+      }).not.toThrow(InstrumentationError);
+    });
+    it('should throw an InstrumentationError when event duration is exactly 8 days after the event occurred', () => {
+      const message = {
+        traits: {
+          action_source: 'some_action_source',
+        },
+        context: {
+          traits: {
+            action_source: 'some_action_source',
+          },
+        },
+        properties: {
+          action_source: 'some_action_source',
+        },
+      };
+      const destination = {
+        ID: 'some_destination_id',
+      };
+      const timeStamp = '2022-01-12T00:00:00Z';
+
+      expect(() => {
+        verifyEventDuration(message, destination, timeStamp);
+      }).toThrow(InstrumentationError);
+    });
+    it('should not throw an InstrumentationError when event duration is greater than 8 days after the event occurred and action_source is physical_store', () => {
+      const message = {
+        traits: {
+          action_source: 'physical_store',
+        },
+        context: {
+          traits: {
+            action_source: 'some_action_source',
+          },
+        },
+        properties: {
+          action_source: 'some_action_source',
+        },
+      };
+      const destination = {
+        ID: 'some_destination_id',
+      };
+      const timeStamp = '2022-01-12T00:00:00Z';
+
+      expect(() => {
+        verifyEventDuration(message, destination, timeStamp);
+      }).not.toThrow(InstrumentationError);
+    });
+  });
 });

src/v0/destinations/google_adwords_offline_conversions/transform.js

@@ -170,9 +170,8 @@ const batchEvents = (storeSalesEvents) => {
       storeSalesEvent.message?.body?.JSON?.addConversionPayload?.operations,
     );
     batchEventResponse.metadatas.push(storeSalesEvent.metadata);
-    batchEventResponse.destination = storeSalesEvent.destination;
   });
-
+  batchEventResponse.destination = storeSalesEvents[0].destination;
   return [
     getSuccessRespEvents(
       batchEventResponse.batchedRequest,

src/v0/destinations/slack/util.js

@@ -83,7 +83,7 @@ const buildDefaultTraitTemplate = (traitsList, traits, template) => {
     generatedStringFromTemplate += `${trait}: {{"${trait}"}} `;
   });
   // else with all traits
-  if (traitsList.length === 0) {
+  if (traitsList.length === 0 && !!traits) {
     Object.keys(traits).forEach((traitKey) => {
       generatedStringFromTemplate += `${traitKey}: {{"${traitKey}"}} `;
     });

src/v0/sources/auth0/mapping.json

@@ -62,5 +62,9 @@
   {
     "sourceKeys": "date",
     "destKeys": ["originalTimestamp", "sentAt"]
+  },
+  {
+    "sourceKeys": "type",
+    "destKeys": "source_type"
   }
 ]

test/integrations/destinations/af/processor/data.ts

@@ -1057,6 +1057,7 @@ export const data = [
                 groupValueTrait: 'age',
                 trackProductsOnce: false,
                 trackRevenuePerProduct: false,
+                afCurrencyAtRoot: true,
               },
             },
           },
@@ -1079,7 +1080,7 @@ export const data = [
               body: {
                 JSON: {
                   eventValue:
-                    '{"properties":{"tax":2,"total":27.5,"coupon":"hasbros","revenue":48,"price":25,"quantity":2,"currency":"ZAR","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"sku":"45790-32","url":"https://www.example.com/product/path","name":"Monopoly: 3rd Edition","price":19,"category":"Games","quantity":1,"image_url":"https:///www.example.com/product/path.jpg","product_id":"507f1f77bcf86cd799439011"},{"sku":"46493-32","name":"Uno Card Game","price":3,"category":"Games","quantity":2,"product_id":"505bd76785ebb509fc183733"}],"shipping":3,"subtotal":22.5,"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f"},"af_revenue":48,"af_quantity":2,"af_price":25}',
+                    '{"properties":{"tax":2,"total":27.5,"coupon":"hasbros","revenue":48,"price":25,"quantity":2,"currency":"ZAR","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"sku":"45790-32","url":"https://www.example.com/product/path","name":"Monopoly: 3rd Edition","price":19,"category":"Games","quantity":1,"image_url":"https:///www.example.com/product/path.jpg","product_id":"507f1f77bcf86cd799439011"},{"sku":"46493-32","name":"Uno Card Game","price":3,"category":"Games","quantity":2,"product_id":"505bd76785ebb509fc183733"}],"shipping":3,"subtotal":22.5,"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f"},"af_revenue":48,"af_quantity":2,"af_price":25,"af_currency":"ZAR"}',
                   eventName: 'normal track event',
                   eventTime: '2020-08-14T05:30:30.118Z',
                   eventCurrency: 'ZAR',

test/integrations/destinations/facebook_conversions/processor/data.ts

@@ -94,7 +94,7 @@ export const data = [
         body: [
           {
             error:
-              'Events must be sent within seven days of their occurrence or up to one minute in the future.',
+              'Events must be sent within 7 days of their occurrence or up to one minute in the future.',
             statusCode: 400,
             statTags: {
               destType: 'FACEBOOK_CONVERSIONS',