Merge branch 'develop' into chore.update-http-request-stat

.github/dependabot.yml

@@ -4,3 +4,11 @@ updates:
     directory: '/'
     schedule:
       interval: 'weekly'
+  - package-ecosystem: 'docker'
+    directory: '/'
+    schedule:
+      interval: 'daily'
+  - package-ecosystem: 'npm'
+    directory: '/'
+    schedule:
+      interval: 'daily'

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.60.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.60.0) (2024-03-20)
+
+
+### Features
+
+* ninetailed: add default value for context.location as {} ([#3197](https://github.com/rudderlabs/rudder-transformer/issues/3197)) ([91fc0fb](https://github.com/rudderlabs/rudder-transformer/commit/91fc0fb3e9eeb127298a0ce305ef6d1d7b72a39f))
+
+
+### Bug Fixes
+
+* heap: make userId as required for track and identify call ([#3198](https://github.com/rudderlabs/rudder-transformer/issues/3198)) ([6a7c534](https://github.com/rudderlabs/rudder-transformer/commit/6a7c534a7df812bb7e39c1905eadcc29d7cd1329))
+* tiktok_ads: validate message.event type ([#3203](https://github.com/rudderlabs/rudder-transformer/issues/3203)) ([a86c277](https://github.com/rudderlabs/rudder-transformer/commit/a86c2771034877cef4161cda61bcda5fdda2d89f))
+
 ## [1.59.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.59.0) (2024-03-18)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.59.0",
+  "version": "1.60.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/cdk/v2/destinations/heap/procWorkflow.yaml

@@ -40,7 +40,9 @@ steps:
       });
       .message.properties.idempotencyKey ? 
         ($.context.payload.idempotency_key = .message.properties.idempotencyKey);
-
+  - name: validateuserId
+    template: |
+      $.assert($.context.payload.identity, "userId is required");
   - name: finalPayload
     description: In batchMode we return payload directly
     condition: $.batchMode

src/cdk/v2/destinations/ninetailed/data/contextMapping.json

@@ -37,7 +37,10 @@
   },
   {
     "sourceKeys": "location",
-    "required": true,
+    "required": false,
+    "metadata": {
+      "defaultValue": {}
+    },
     "destKey": "location"
   }
 ]

src/constants/destinationCanonicalNames.js

@@ -152,6 +152,18 @@ const DestCanonicalNames = {
     'the trade desk',
   ],
   INTERCOM: ['INTERCOM', 'intercom', 'Intercom'],
+  GOOGLE_ADWORDS_REMARKETING_LISTS: [
+    'GOOGLE_ADWORDS_REMARKETING_LISTS',
+    'google_adwords_remarketing_lists',
+    'Google Adwords Remarketing Lists',
+    'google adwords remarketing lists',
+  ],
+  GOOGLE_ADWORDS_OFFLINE_CONVERSIONS: [
+    'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS',
+    'google_adwords_offline_conversions',
+    'Google Adwords Offline Conversions',
+    'google adwords offline conversions',
+  ],
   koala: ['Koala', 'koala', 'KOALA'],
 };
 

src/features.json

@@ -85,5 +85,5 @@
     "SPRIG"
   ],
   "supportSourceTransformV1": true,
-  "supportTransformerProxyV1": false
+  "supportTransformerProxyV1": true
 }

src/types/zodTypes.ts

@@ -1,5 +1,5 @@
-import { z } from 'zod';
 import { isDefinedAndNotNullAndNotEmpty } from '@rudderstack/integrations-lib';
+import { z } from 'zod';
 import { isHttpStatusSuccess } from '../v0/util';
 
 const ProcessorTransformationOutputSchema = z.object({

src/util/prometheus.js

@@ -575,6 +575,30 @@ class Prometheus {
         type: 'gauge',
         labelNames: ['destination_id'],
       },
+      {
+        name: 'mixpanel_batch_engage_pack_size',
+        help: 'mixpanel_batch_engage_pack_size',
+        type: 'gauge',
+        labelNames: ['destination_id'],
+      },
+      {
+        name: 'mixpanel_batch_group_pack_size',
+        help: 'mixpanel_batch_group_pack_size',
+        type: 'gauge',
+        labelNames: ['destination_id'],
+      },
+      {
+        name: 'mixpanel_batch_track_pack_size',
+        help: 'mixpanel_batch_track_pack_size',
+        type: 'gauge',
+        labelNames: ['destination_id'],
+      },
+      {
+        name: 'mixpanel_batch_import_pack_size',
+        help: 'mixpanel_batch_import_pack_size',
+        type: 'gauge',
+        labelNames: ['destination_id'],
+      },
 
       // Histograms
       {

src/v0/destinations/google_adwords_offline_conversions/config.js

@@ -1,6 +1,6 @@
 const { getMappingConfig } = require('../../util');
 
-const API_VERSION = 'v14';
+const API_VERSION = 'v16';
 
 const BASE_ENDPOINT = `https://googleads.googleapis.com/${API_VERSION}/customers/:customerId`;
 
@@ -42,6 +42,11 @@ const CONVERSION_CUSTOM_VARIABLE_CACHE_TTL = process.env.CONVERSION_CUSTOM_VARIA
 
 const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname);
 
+const consentConfigMap = {
+  personalizationConsent: 'adPersonalization',
+  userDataConsent: 'adUserData',
+};
+
 module.exports = {
   trackClickConversionsMapping:
     MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK_CLICK_CONVERSIONS_CONFIG.name],
@@ -58,4 +63,5 @@ module.exports = {
     MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK_STORE_CONVERSION_CONFIG_ADD_CONVERSION.name],
   trackAddStoreAddressConversionsMapping:
     MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK_STORE_ADDRESS_IDENTIFIER.name],
+  consentConfigMap,
 };

src/v0/destinations/google_adwords_offline_conversions/transform.js

@@ -3,24 +3,21 @@ const { InstrumentationError, ConfigurationError } = require('@rudderstack/integ
 const { EventType } = require('../../../constants');
 const {
   getHashFromArrayWithDuplicate,
-  constructPayload,
   removeHyphens,
   getHashFromArray,
   handleRtTfSingleEventError,
   defaultBatchRequestConfig,
   getSuccessRespEvents,
   combineBatchRequestsWithSameJobIds,
 } = require('../../util');
-const {
-  CALL_CONVERSION,
-  trackCallConversionsMapping,
-  STORE_CONVERSION_CONFIG,
-} = require('./config');
+const { CALL_CONVERSION, STORE_CONVERSION_CONFIG } = require('./config');
 const {
   validateDestinationConfig,
   getStoreConversionPayload,
   requestBuilder,
   getClickConversionPayloadAndEndpoint,
+  getConsentsDataFromIntegrationObj,
+  getCallConversionPayload,
 } = require('./utils');
 const helper = require('./helper');
 
@@ -41,12 +38,15 @@ const getConversions = (message, metadata, { Config }, event, conversionType) =>
   const { properties, timestamp, originalTimestamp } = message;
 
   const filteredCustomerId = removeHyphens(customerId);
+  const eventLevelConsentsData = getConsentsDataFromIntegrationObj(message);
+
   if (conversionType === 'click') {
     // click conversion
     const convertedPayload = getClickConversionPayloadAndEndpoint(
       message,
       Config,
       filteredCustomerId,
+      eventLevelConsentsData,
     );
     payload = convertedPayload.payload;
     endpoint = convertedPayload.endpoint;
@@ -55,7 +55,7 @@ const getConversions = (message, metadata, { Config }, event, conversionType) =>
     endpoint = STORE_CONVERSION_CONFIG.replace(':customerId', filteredCustomerId);
   } else {
     // call conversions
-    payload = constructPayload(message, trackCallConversionsMapping);
+    payload = getCallConversionPayload(message, Config, eventLevelConsentsData);
     endpoint = CALL_CONVERSION.replace(':customerId', filteredCustomerId);
   }
 
@@ -119,7 +119,6 @@ const trackResponseBuilder = (message, metadata, destination) => {
 
 const process = async (event) => {
   const { message, metadata, destination } = event;
-
   if (!message.type) {
     throw new InstrumentationError('Message type is not present. Aborting message.');
   }

src/v0/destinations/google_adwords_offline_conversions/utils.js

@@ -18,6 +18,7 @@ const {
   isDefinedAndNotNull,
   getAuthErrCategoryFromStCode,
   getAccessToken,
+  getIntegrationsObj,
 } = require('../../util');
 const {
   SEARCH_STREAM,
@@ -27,10 +28,13 @@ const {
   trackAddStoreAddressConversionsMapping,
   trackClickConversionsMapping,
   CLICK_CONVERSION,
+  trackCallConversionsMapping,
+  consentConfigMap,
 } = require('./config');
 const { processAxiosResponse } = require('../../../adapters/utils/networkUtils');
 const Cache = require('../../util/cache');
 const helper = require('./helper');
+const { finaliseConsent } = require('../../util/googleUtils');
 
 const conversionActionIdCache = new Cache(CONVERSION_ACTION_ID_CACHE_TTL);
 
@@ -221,6 +225,17 @@ function getExisitingUserIdentifier(userIdentifierInfo, defaultUserIdentifier) {
   return result;
 }
 
+const getCallConversionPayload = (message, Config, eventLevelConsentsData) => {
+  const payload = constructPayload(message, trackCallConversionsMapping);
+  // here conversions[0] should be present because there are some mandatory properties mapped in the mapping json.
+  payload.conversions[0].consent = finaliseConsent(
+    consentConfigMap,
+    eventLevelConsentsData,
+    Config,
+  );
+  return payload;
+};
+
 /**
  * This Function create the add conversion payload
  * and returns the payload
@@ -277,6 +292,10 @@ const getAddConversionPayload = (message, Config) => {
       set(payload, 'operations.create.userIdentifiers[0]', {});
     }
   }
+  // add consent support for store conversions. Note: No event level consent supported.
+  const consentObject = finaliseConsent(consentConfigMap, {}, Config);
+  // create property should be present because there are some mandatory properties mapped in the mapping json.
+  set(payload, 'operations.create.consent', consentObject);
   return payload;
 };
 
@@ -292,7 +311,12 @@ const getStoreConversionPayload = (message, Config, event) => {
   return payload;
 };
 
-const getClickConversionPayloadAndEndpoint = (message, Config, filteredCustomerId) => {
+const getClickConversionPayloadAndEndpoint = (
+  message,
+  Config,
+  filteredCustomerId,
+  eventLevelConsent,
+) => {
   const email = getFieldValueFromMessage(message, 'emailOnly');
   const phone = getFieldValueFromMessage(message, 'phone');
   const { hashUserIdentifier, defaultUserIdentifier, UserIdentifierSource, conversionEnvironment } =
@@ -364,9 +388,19 @@ const getClickConversionPayloadAndEndpoint = (message, Config, filteredCustomerI
   if (!properties.conversionEnvironment && conversionEnvironment !== 'none') {
     set(payload, 'conversions[0].conversionEnvironment', conversionEnvironment);
   }
+
+  // add consent support for click conversions
+  const consentObject = finaliseConsent(consentConfigMap, eventLevelConsent, Config);
+  // here conversions[0] is expected to be present there are some mandatory properties mapped in the mapping json.
+  set(payload, 'conversions[0].consent', consentObject);
   return { payload, endpoint };
 };
 
+const getConsentsDataFromIntegrationObj = (message) => {
+  const integrationObj = getIntegrationsObj(message, 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS') || {};
+  return integrationObj?.consents || {};
+};
+
 module.exports = {
   validateDestinationConfig,
   generateItemListFromProducts,
@@ -377,4 +411,6 @@ module.exports = {
   buildAndGetAddress,
   getClickConversionPayloadAndEndpoint,
   getExisitingUserIdentifier,
+  getConsentsDataFromIntegrationObj,
+  getCallConversionPayload,
 };