Merge branch 'develop' of https://github.com/rudderlabs/rudder-transf…

…ormer into chore.webhook-tests

.github/dependabot.yml

@@ -4,3 +4,11 @@ updates:
     directory: '/'
     schedule:
       interval: 'weekly'
+  - package-ecosystem: 'docker'
+    directory: '/'
+    schedule:
+      interval: 'daily'
+  - package-ecosystem: 'npm'
+    directory: '/'
+    schedule:
+      interval: 'daily'

CHANGELOG.md

@@ -2,6 +2,41 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.60.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.60.0) (2024-03-20)
+
+
+### Features
+
+* ninetailed: add default value for context.location as {} ([#3197](https://github.com/rudderlabs/rudder-transformer/issues/3197)) ([91fc0fb](https://github.com/rudderlabs/rudder-transformer/commit/91fc0fb3e9eeb127298a0ce305ef6d1d7b72a39f))
+
+
+### Bug Fixes
+
+* heap: make userId as required for track and identify call ([#3198](https://github.com/rudderlabs/rudder-transformer/issues/3198)) ([6a7c534](https://github.com/rudderlabs/rudder-transformer/commit/6a7c534a7df812bb7e39c1905eadcc29d7cd1329))
+* tiktok_ads: validate message.event type ([#3203](https://github.com/rudderlabs/rudder-transformer/issues/3203)) ([a86c277](https://github.com/rudderlabs/rudder-transformer/commit/a86c2771034877cef4161cda61bcda5fdda2d89f))
+
+## [1.59.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.59.0) (2024-03-18)
+
+
+### Features
+
+* add Koala destination ([#3122](https://github.com/rudderlabs/rudder-transformer/issues/3122)) ([1ca039d](https://github.com/rudderlabs/rudder-transformer/commit/1ca039d64ebb1a18a0fc6b78ed5ee08528ad6b48))
+* add support of skip_user_properties_sync on Amplitude ([#3181](https://github.com/rudderlabs/rudder-transformer/issues/3181)) ([5e4ddbd](https://github.com/rudderlabs/rudder-transformer/commit/5e4ddbd8a591341a581a5721505d6dcb010f2eec))
+* adding zod validations ([#3066](https://github.com/rudderlabs/rudder-transformer/issues/3066)) ([325433b](https://github.com/rudderlabs/rudder-transformer/commit/325433b9188c8d1dbe740c7e193cdc2e58fdd751))
+* onboard destination movable ink ([#3167](https://github.com/rudderlabs/rudder-transformer/issues/3167)) ([7018b1e](https://github.com/rudderlabs/rudder-transformer/commit/7018b1e5e7f37ae177191c5ecf3a71cfe2f3d147))
+* update proxy tests for cm360 ([#3039](https://github.com/rudderlabs/rudder-transformer/issues/3039)) ([0504ffa](https://github.com/rudderlabs/rudder-transformer/commit/0504ffa898956f5b61771fb32ecfd0e0bf15248f))
+* use dontBatch directive in algolia ([#3169](https://github.com/rudderlabs/rudder-transformer/issues/3169)) ([916aaec](https://github.com/rudderlabs/rudder-transformer/commit/916aaecb1939160620d5fd3c4c0c0e33f2a371b2))
+
+
+### Bug Fixes
+
+* api contract for v1 proxy ([#3049](https://github.com/rudderlabs/rudder-transformer/issues/3049)) ([93947db](https://github.com/rudderlabs/rudder-transformer/commit/93947db35cdaf1ca7ed87ec5f73567754af312ab))
+* email mapping for clevertap ([#3173](https://github.com/rudderlabs/rudder-transformer/issues/3173)) ([04eab92](https://github.com/rudderlabs/rudder-transformer/commit/04eab92e1c383f9e8cdd5c845530a42a0af2932a))
+* fb pixel test case refactor ([#3075](https://github.com/rudderlabs/rudder-transformer/issues/3075)) ([cff7d1c](https://github.com/rudderlabs/rudder-transformer/commit/cff7d1c4578087a37614c0ef4529058481873479))
+* fixed 500 status for algolia dontBatch ([#3178](https://github.com/rudderlabs/rudder-transformer/issues/3178)) ([6330888](https://github.com/rudderlabs/rudder-transformer/commit/6330888ad5c67e3a800037b56501fc08da09e4d1))
+* label not present in prometheus metrics ([#3176](https://github.com/rudderlabs/rudder-transformer/issues/3176)) ([01d460c](https://github.com/rudderlabs/rudder-transformer/commit/01d460c3edaf39b35c4686516c9e9140be46aa5e))
+* send proper status to server in cm360 ([#3127](https://github.com/rudderlabs/rudder-transformer/issues/3127)) ([229ce47](https://github.com/rudderlabs/rudder-transformer/commit/229ce473af1ddd62d946bea1b018c882b142a5ef))
+
 ## [1.58.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.58.0) (2024-03-04)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.58.0",
+  "version": "1.60.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/cdk/v2/destinations/heap/procWorkflow.yaml

@@ -40,7 +40,9 @@ steps:
       });
       .message.properties.idempotencyKey ? 
         ($.context.payload.idempotency_key = .message.properties.idempotencyKey);
-
+  - name: validateuserId
+    template: |
+      $.assert($.context.payload.identity, "userId is required");
   - name: finalPayload
     description: In batchMode we return payload directly
     condition: $.batchMode

src/cdk/v2/destinations/movable_ink/procWorkflow.yaml

@@ -33,6 +33,7 @@ steps:
       );
 
       $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting");
+      $.validateEventPayload(.message);
 
   - name: preparePayload
     description: Prepare payload for identify and track. This payload schema needs to be configured in the Movable Ink dashboard. Movable Ink will discard any additional fields from the input payload.

src/cdk/v2/destinations/movable_ink/utils.js

@@ -0,0 +1,21 @@
+const { InstrumentationError } = require('@rudderstack/integrations-lib');
+
+const validateEventPayload = (message) => {
+  const { event } = message;
+  const { properties } = message;
+  if (event === 'Products Searched' && !properties?.query) {
+    throw new InstrumentationError("Missing 'query' property in properties. Aborting");
+  }
+
+  if (
+    (event === 'Product Added' ||
+      event === 'Product Removed' ||
+      event === 'Product Viewed' ||
+      event === 'Category Viewed') &&
+    !properties?.product_id
+  ) {
+    throw new InstrumentationError("Missing 'product_id' property in properties. Aborting");
+  }
+};
+
+module.exports = { validateEventPayload };

src/cdk/v2/destinations/ninetailed/data/contextMapping.json

@@ -37,7 +37,10 @@
   },
   {
     "sourceKeys": "location",
-    "required": true,
+    "required": false,
+    "metadata": {
+      "defaultValue": {}
+    },
     "destKey": "location"
   }
 ]

src/constants/destinationCanonicalNames.js

@@ -152,6 +152,18 @@ const DestCanonicalNames = {
     'the trade desk',
   ],
   INTERCOM: ['INTERCOM', 'intercom', 'Intercom'],
+  GOOGLE_ADWORDS_REMARKETING_LISTS: [
+    'GOOGLE_ADWORDS_REMARKETING_LISTS',
+    'google_adwords_remarketing_lists',
+    'Google Adwords Remarketing Lists',
+    'google adwords remarketing lists',
+  ],
+  GOOGLE_ADWORDS_OFFLINE_CONVERSIONS: [
+    'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS',
+    'google_adwords_offline_conversions',
+    'Google Adwords Offline Conversions',
+    'google adwords offline conversions',
+  ],
   koala: ['Koala', 'koala', 'KOALA'],
 };
 

src/features.json

@@ -85,5 +85,5 @@
     "SPRIG"
   ],
   "supportSourceTransformV1": true,
-  "supportTransformerProxyV1": false
+  "supportTransformerProxyV1": true
 }

src/v0/destinations/google_adwords_offline_conversions/config.js

@@ -1,6 +1,6 @@
 const { getMappingConfig } = require('../../util');
 
-const API_VERSION = 'v14';
+const API_VERSION = 'v16';
 
 const BASE_ENDPOINT = `https://googleads.googleapis.com/${API_VERSION}/customers/:customerId`;
 
@@ -42,6 +42,11 @@ const CONVERSION_CUSTOM_VARIABLE_CACHE_TTL = process.env.CONVERSION_CUSTOM_VARIA
 
 const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname);
 
+const consentConfigMap = {
+  personalizationConsent: 'adPersonalization',
+  userDataConsent: 'adUserData',
+};
+
 module.exports = {
   trackClickConversionsMapping:
     MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK_CLICK_CONVERSIONS_CONFIG.name],
@@ -58,4 +63,5 @@ module.exports = {
     MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK_STORE_CONVERSION_CONFIG_ADD_CONVERSION.name],
   trackAddStoreAddressConversionsMapping:
     MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK_STORE_ADDRESS_IDENTIFIER.name],
+  consentConfigMap,
 };

src/v0/destinations/google_adwords_offline_conversions/transform.js

@@ -3,24 +3,21 @@ const { InstrumentationError, ConfigurationError } = require('@rudderstack/integ
 const { EventType } = require('../../../constants');
 const {
   getHashFromArrayWithDuplicate,
-  constructPayload,
   removeHyphens,
   getHashFromArray,
   handleRtTfSingleEventError,
   defaultBatchRequestConfig,
   getSuccessRespEvents,
   combineBatchRequestsWithSameJobIds,
 } = require('../../util');
-const {
-  CALL_CONVERSION,
-  trackCallConversionsMapping,
-  STORE_CONVERSION_CONFIG,
-} = require('./config');
+const { CALL_CONVERSION, STORE_CONVERSION_CONFIG } = require('./config');
 const {
   validateDestinationConfig,
   getStoreConversionPayload,
   requestBuilder,
   getClickConversionPayloadAndEndpoint,
+  getConsentsDataFromIntegrationObj,
+  getCallConversionPayload,
 } = require('./utils');
 const helper = require('./helper');
 
@@ -41,12 +38,15 @@ const getConversions = (message, metadata, { Config }, event, conversionType) =>
   const { properties, timestamp, originalTimestamp } = message;
 
   const filteredCustomerId = removeHyphens(customerId);
+  const eventLevelConsentsData = getConsentsDataFromIntegrationObj(message);
+
   if (conversionType === 'click') {
     // click conversion
     const convertedPayload = getClickConversionPayloadAndEndpoint(
       message,
       Config,
       filteredCustomerId,
+      eventLevelConsentsData,
     );
     payload = convertedPayload.payload;
     endpoint = convertedPayload.endpoint;
@@ -55,7 +55,7 @@ const getConversions = (message, metadata, { Config }, event, conversionType) =>
     endpoint = STORE_CONVERSION_CONFIG.replace(':customerId', filteredCustomerId);
   } else {
     // call conversions
-    payload = constructPayload(message, trackCallConversionsMapping);
+    payload = getCallConversionPayload(message, Config, eventLevelConsentsData);
     endpoint = CALL_CONVERSION.replace(':customerId', filteredCustomerId);
   }
 
@@ -119,7 +119,6 @@ const trackResponseBuilder = (message, metadata, destination) => {
 
 const process = async (event) => {
   const { message, metadata, destination } = event;
-
   if (!message.type) {
     throw new InstrumentationError('Message type is not present. Aborting message.');
   }

src/v0/destinations/google_adwords_offline_conversions/utils.js

@@ -18,6 +18,7 @@ const {
   isDefinedAndNotNull,
   getAuthErrCategoryFromStCode,
   getAccessToken,
+  getIntegrationsObj,
 } = require('../../util');
 const {
   SEARCH_STREAM,
@@ -27,10 +28,13 @@ const {
   trackAddStoreAddressConversionsMapping,
   trackClickConversionsMapping,
   CLICK_CONVERSION,
+  trackCallConversionsMapping,
+  consentConfigMap,
 } = require('./config');
 const { processAxiosResponse } = require('../../../adapters/utils/networkUtils');
 const Cache = require('../../util/cache');
 const helper = require('./helper');
+const { finaliseConsent } = require('../../util/googleUtils');
 
 const conversionActionIdCache = new Cache(CONVERSION_ACTION_ID_CACHE_TTL);
 
@@ -221,6 +225,17 @@ function getExisitingUserIdentifier(userIdentifierInfo, defaultUserIdentifier) {
   return result;
 }
 
+const getCallConversionPayload = (message, Config, eventLevelConsentsData) => {
+  const payload = constructPayload(message, trackCallConversionsMapping);
+  // here conversions[0] should be present because there are some mandatory properties mapped in the mapping json.
+  payload.conversions[0].consent = finaliseConsent(
+    consentConfigMap,
+    eventLevelConsentsData,
+    Config,
+  );
+  return payload;
+};
+
 /**
  * This Function create the add conversion payload
  * and returns the payload
@@ -277,6 +292,10 @@ const getAddConversionPayload = (message, Config) => {
       set(payload, 'operations.create.userIdentifiers[0]', {});
     }
   }
+  // add consent support for store conversions. Note: No event level consent supported.
+  const consentObject = finaliseConsent(consentConfigMap, {}, Config);
+  // create property should be present because there are some mandatory properties mapped in the mapping json.
+  set(payload, 'operations.create.consent', consentObject);
   return payload;
 };
 
@@ -292,7 +311,12 @@ const getStoreConversionPayload = (message, Config, event) => {
   return payload;
 };
 
-const getClickConversionPayloadAndEndpoint = (message, Config, filteredCustomerId) => {
+const getClickConversionPayloadAndEndpoint = (
+  message,
+  Config,
+  filteredCustomerId,
+  eventLevelConsent,
+) => {
   const email = getFieldValueFromMessage(message, 'emailOnly');
   const phone = getFieldValueFromMessage(message, 'phone');
   const { hashUserIdentifier, defaultUserIdentifier, UserIdentifierSource, conversionEnvironment } =
@@ -364,9 +388,19 @@ const getClickConversionPayloadAndEndpoint = (message, Config, filteredCustomerI
   if (!properties.conversionEnvironment && conversionEnvironment !== 'none') {
     set(payload, 'conversions[0].conversionEnvironment', conversionEnvironment);
   }
+
+  // add consent support for click conversions
+  const consentObject = finaliseConsent(consentConfigMap, eventLevelConsent, Config);
+  // here conversions[0] is expected to be present there are some mandatory properties mapped in the mapping json.
+  set(payload, 'conversions[0].consent', consentObject);
   return { payload, endpoint };
 };
 
+const getConsentsDataFromIntegrationObj = (message) => {
+  const integrationObj = getIntegrationsObj(message, 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS') || {};
+  return integrationObj?.consents || {};
+};
+
 module.exports = {
   validateDestinationConfig,
   generateItemListFromProducts,
@@ -377,4 +411,6 @@ module.exports = {
   buildAndGetAddress,
   getClickConversionPayloadAndEndpoint,
   getExisitingUserIdentifier,
+  getConsentsDataFromIntegrationObj,
+  getCallConversionPayload,
 };