feat: consent field support for gaoc for API v15 and upgrade the api version from v14 to v16

src/constants/destinationCanonicalNames.js

@@ -152,6 +152,18 @@ const DestCanonicalNames = {
     'the trade desk',
   ],
   INTERCOM: ['INTERCOM', 'intercom', 'Intercom'],
+  GOOGLE_ADWORDS_REMARKETING_LISTS: [
+    'GOOGLE_ADWORDS_REMARKETING_LISTS',
+    'google_adwords_remarketing_lists',
+    'Google Adwords Remarketing Lists',
+    'google adwords remarketing lists',
+  ],
+  GOOGLE_ADWORDS_OFFLINE_CONVERSIONS: [
+    'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS',
+    'google_adwords_offline_conversions',
+    'Google Adwords Offline Conversions',
+    'google adwords offline conversions',
+  ],
   koala: ['Koala', 'koala', 'KOALA'],
 };
 

src/v0/destinations/google_adwords_offline_conversions/config.js

@@ -1,6 +1,6 @@
 const { getMappingConfig } = require('../../util');
 
-const API_VERSION = 'v14';
+const API_VERSION = 'v16';
 
 const BASE_ENDPOINT = `https://googleads.googleapis.com/${API_VERSION}/customers/:customerId`;
 

src/v0/destinations/google_adwords_offline_conversions/transform.js

@@ -3,24 +3,21 @@ const { InstrumentationError, ConfigurationError } = require('@rudderstack/integ
 const { EventType } = require('../../../constants');
 const {
   getHashFromArrayWithDuplicate,
-  constructPayload,
   removeHyphens,
   getHashFromArray,
   handleRtTfSingleEventError,
   defaultBatchRequestConfig,
   getSuccessRespEvents,
   combineBatchRequestsWithSameJobIds,
 } = require('../../util');
-const {
-  CALL_CONVERSION,
-  trackCallConversionsMapping,
-  STORE_CONVERSION_CONFIG,
-} = require('./config');
+const { CALL_CONVERSION, STORE_CONVERSION_CONFIG } = require('./config');
 const {
   validateDestinationConfig,
   getStoreConversionPayload,
   requestBuilder,
   getClickConversionPayloadAndEndpoint,
+  getConsentsDataFromIntegrationObj,
+  getCallConversionPayload,
 } = require('./utils');
 const helper = require('./helper');
 
@@ -41,12 +38,15 @@ const getConversions = (message, metadata, { Config }, event, conversionType) =>
   const { properties, timestamp, originalTimestamp } = message;
 
   const filteredCustomerId = removeHyphens(customerId);
+  const eventLevelConsentsData = getConsentsDataFromIntegrationObj(message);
+
   if (conversionType === 'click') {
     // click conversion
     const convertedPayload = getClickConversionPayloadAndEndpoint(
       message,
       Config,
       filteredCustomerId,
+      eventLevelConsentsData,
     );
     payload = convertedPayload.payload;
     endpoint = convertedPayload.endpoint;
@@ -55,7 +55,7 @@ const getConversions = (message, metadata, { Config }, event, conversionType) =>
     endpoint = STORE_CONVERSION_CONFIG.replace(':customerId', filteredCustomerId);
   } else {
     // call conversions
-    payload = constructPayload(message, trackCallConversionsMapping);
+    payload = { ...getCallConversionPayload(message, Config, eventLevelConsentsData) };
     endpoint = CALL_CONVERSION.replace(':customerId', filteredCustomerId);
   }
 
@@ -119,7 +119,6 @@ const trackResponseBuilder = (message, metadata, destination) => {
 
 const process = async (event) => {
   const { message, metadata, destination } = event;
-
   if (!message.type) {
     throw new InstrumentationError('Message type is not present. Aborting message.');
   }

src/v0/destinations/google_adwords_offline_conversions/utils.js

@@ -17,6 +17,7 @@ const {
   isDefinedAndNotNull,
   getAuthErrCategoryFromStCode,
   getAccessToken,
+  getIntegrationsObj,
 } = require('../../util');
 const {
   SEARCH_STREAM,
@@ -26,10 +27,12 @@ const {
   trackAddStoreAddressConversionsMapping,
   trackClickConversionsMapping,
   CLICK_CONVERSION,
+  trackCallConversionsMapping,
 } = require('./config');
 const { processAxiosResponse } = require('../../../adapters/utils/networkUtils');
 const Cache = require('../../util/cache');
 const helper = require('./helper');
+const { finaliseConsent } = require('../../util/googleUtils');
 
 const conversionActionIdCache = new Cache(CONVERSION_ACTION_ID_CACHE_TTL);
 
@@ -216,6 +219,13 @@ function getExisitingUserIdentifier(userIdentifierInfo, defaultUserIdentifier) {
   return result;
 }
 
+const getCallConversionPayload = (message, Config, eventLevelConsentsData) => {
+  const payload = constructPayload(message, trackCallConversionsMapping);
+  // here conversions[0] should be present because there are some mandatory properties mapped in the mapping json.
+  payload.conversions[0].consent = finaliseConsent(eventLevelConsentsData, Config);
+  return payload;
+};
+
 /**
  * This Function create the add conversion payload
  * and returns the payload
@@ -272,6 +282,10 @@ const getAddConversionPayload = (message, Config) => {
       set(payload, 'operations.create.userIdentifiers[0]', {});
     }
   }
+  // add consent support for store conversions. Note: No event level consent supported.
+  const consentObject = finaliseConsent({}, Config);
+  // create property should be present because there are some mandatory properties mapped in the mapping json.
+  set(payload, 'operations.create.consent', consentObject);
   return payload;
 };
 
@@ -287,7 +301,12 @@ const getStoreConversionPayload = (message, Config, event) => {
   return payload;
 };
 
-const getClickConversionPayloadAndEndpoint = (message, Config, filteredCustomerId) => {
+const getClickConversionPayloadAndEndpoint = (
+  message,
+  Config,
+  filteredCustomerId,
+  eventLevelConsent,
+) => {
   const email = getFieldValueFromMessage(message, 'emailOnly');
   const phone = getFieldValueFromMessage(message, 'phone');
   const { hashUserIdentifier, defaultUserIdentifier, UserIdentifierSource, conversionEnvironment } =
@@ -359,9 +378,19 @@ const getClickConversionPayloadAndEndpoint = (message, Config, filteredCustomerI
   if (!properties.conversionEnvironment && conversionEnvironment !== 'none') {
     set(payload, 'conversions[0].conversionEnvironment', conversionEnvironment);
   }
+
+  // add consent support for click conversions
+  const consentObject = finaliseConsent(eventLevelConsent, Config);
+  // here conversions[0] is expected to be present there are some mandatory properties mapped in the mapping json.
+  set(payload, 'conversions[0].consent', consentObject);
   return { payload, endpoint };
 };
 
+const getConsentsDataFromIntegrationObj = (message) => {
+  const integrationObj = getIntegrationsObj(message, 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS') || {};
+  return integrationObj?.consents || {};
+};
+
 module.exports = {
   validateDestinationConfig,
   generateItemListFromProducts,
@@ -372,4 +401,6 @@ module.exports = {
   buildAndGetAddress,
   getClickConversionPayloadAndEndpoint,
   getExisitingUserIdentifier,
+  getConsentsDataFromIntegrationObj,
+  getCallConversionPayload,
 };

src/v0/destinations/google_adwords_offline_conversions/utils.test.js

@@ -2,6 +2,7 @@ const {
   getClickConversionPayloadAndEndpoint,
   buildAndGetAddress,
   getExisitingUserIdentifier,
+  getConsentsDataFromIntegrationObj,
 } = require('./utils');
 
 const getTestMessage = () => {
@@ -161,12 +162,16 @@ describe('getExisitingUserIdentifier util tests', () => {
 describe('getClickConversionPayloadAndEndpoint util tests', () => {
   it('getClickConversionPayloadAndEndpoint flow check when default field identifier is present', () => {
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {
             conversionDateTime: '2022-01-01 12:32:45-08:00',
             conversionEnvironment: 'WEB',
+            consent: {
+              adPersonalization: 'UNSPECIFIED',
+              adUserData: 'UNSPECIFIED',
+            },
             userIdentifiers: [
               {
                 hashedEmail: 'fa922cb41ff930664d4c9ced3c472ce7ecf29a0f8248b7018456e990177fff75',
@@ -187,11 +192,15 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
     delete fittingPayload.traits.email;
     delete fittingPayload.properties.email;
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {
             conversionDateTime: '2022-01-01 12:32:45-08:00',
+            consent: {
+              adPersonalization: 'UNSPECIFIED',
+              adUserData: 'UNSPECIFIED',
+            },
             conversionEnvironment: 'WEB',
             userIdentifiers: [
               {
@@ -215,7 +224,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
     delete fittingPayload.traits.phone;
     delete fittingPayload.properties.email;
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {
@@ -237,7 +246,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
     ).toThrow('Either of email or phone is required for user identifier');
   });
 
-  it('getClickConversionPayloadAndEndpoint flow check when default field identifier is present and product list present', () => {
+  it('finaliseConsent', () => {
     let fittingPayload = { ...getTestMessage() };
     fittingPayload.properties.products = [
       {
@@ -251,13 +260,17 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
       },
     ];
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {
             cartData: { items: [{ productId: 1234, quantity: 2, unitPrice: 10 }] },
             conversionDateTime: '2022-01-01 12:32:45-08:00',
             conversionEnvironment: 'WEB',
+            consent: {
+              adPersonalization: 'UNSPECIFIED',
+              adUserData: 'UNSPECIFIED',
+            },
             userIdentifiers: [
               {
                 hashedEmail: 'fa922cb41ff930664d4c9ced3c472ce7ecf29a0f8248b7018456e990177fff75',
@@ -273,3 +286,29 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
     );
   });
 });
+
+describe('getConsentsDataFromIntegrationObj', () => {
+  it('should return an empty object when conversionType is "store"', () => {
+    const message = {};
+    const result = getConsentsDataFromIntegrationObj(message);
+    expect(result).toEqual({});
+  });
+  it('should return the consent object when conversion type is call', () => {
+    const message = {
+      integrations: {
+        GOOGLE_ADWORDS_OFFLINE_CONVERSIONS: {
+          consents: {
+            adUserData: 'GRANTED',
+            adPersonalization: 'DENIED',
+          },
+        },
+      },
+    };
+    const conversionType = 'call';
+    const result = getConsentsDataFromIntegrationObj(message, conversionType);
+    expect(result).toEqual({
+      adPersonalization: 'DENIED',
+      adUserData: 'GRANTED',
+    });
+  });
+});

src/v0/destinations/google_adwords_remarketing_lists/transform.js

@@ -15,7 +15,7 @@ const {
   getAccessToken,
 } = require('../../util');
 
-const { populateConsentForGoogleDestinations } = require('../../util/googleUtils');
+const { populateConsentFromConfig } = require('../../util/googleUtils');
 
 const {
   offlineDataJobsMapping,
@@ -218,7 +218,7 @@ const processEvent = async (metadata, message, destination) => {
     }
 
     Object.values(createdPayload).forEach((data) => {
-      const consentObj = populateConsentForGoogleDestinations(destination.Config);
+      const consentObj = populateConsentFromConfig(destination.Config);
       response.push(responseBuilder(metadata, data, destination, message, consentObj));
     });
     return response;

src/v0/util/googleUtils/index.js

@@ -1,36 +1,89 @@
 const GOOGLE_ALLOWED_CONSENT_STATUS = ['UNSPECIFIED', 'UNKNOWN', 'GRANTED', 'DENIED'];
 
+const UNSPECIFIED_CONSENT = 'UNSPECIFIED';
+const UNKNOWN_CONSENT = 'UNKNOWN';
+const ADWORDS_CONSENT_FILEDS = ['adUserData', 'adPersonalization'];
+
 /**
  * Populates the consent object based on the provided properties.
  *
  * @param {object} properties - message.properties containing properties related to consent.
  * @returns {object} - An object containing consent information.
- * ref : https://developers.google.com/google-ads/api/rest/reference/rest/v15/Consent
+ * ref : https://developers.google.com/google-ads/api/rest/reference/rest/v16/Consent
  */
 
-const populateConsentForGoogleDestinations = (config) => {
+const populateConsentFromConfig = (config) => {
   const consent = {};
 
   if (config?.userDataConsent) {
     if (GOOGLE_ALLOWED_CONSENT_STATUS.includes(config.userDataConsent)) {
       consent.adUserData = config.userDataConsent;
     } else {
-      consent.adUserData = 'UNKNOWN';
+      consent.adUserData = UNKNOWN_CONSENT;
     }
   } else {
-    consent.adUserData = 'UNSPECIFIED';
+    consent.adUserData = UNSPECIFIED_CONSENT;
   }
 
   if (config?.personalizationConsent) {
     if (GOOGLE_ALLOWED_CONSENT_STATUS.includes(config.personalizationConsent)) {
       consent.adPersonalization = config.personalizationConsent;
     } else {
-      consent.adPersonalization = 'UNKNOWN';
+      consent.adPersonalization = UNKNOWN_CONSENT;
     }
   } else {
-    consent.adPersonalization = 'UNSPECIFIED';
+    consent.adPersonalization = UNSPECIFIED_CONSENT;
   }
   return consent;
 };
 
-module.exports = { populateConsentForGoogleDestinations };
+/**
+ * Populates the consent object based on the provided properties.
+ *
+ * @param {object} properties - message.properties containing properties related to consent.
+ * @returns {object} - An object containing consent information.
+ *  * ref :
+ * 1) For click conversion :
+ *  a) https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/uploadClickConversions#ClickConversion
+ *  b) https://developers.google.com/google-ads/api/reference/rpc/v16/ClickConversion#consent
+ * 2) For Call conversion :
+ *  a) https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/uploadCallConversions#CallConversion
+ *  b) https://developers.google.com/google-ads/api/reference/rpc/v16/CallConversion#consent
+ * 3) For Store sales conversion :
+ *  a) https://developers.google.com/google-ads/api/reference/rpc/v16/UserData
+ *  b) https://developers.google.com/google-ads/api/reference/rpc/v16/UserData#consent
+ */
+
+const finaliseConsent = (eventLevelConsent, destConfig) => {
+  const consents = eventLevelConsent || {};
+
+  const defaultConsentBlock = populateConsentFromConfig(destConfig);
+
+  // Define a function to process consent based on type
+  const processConsent = (consentType) => {
+    if (!consents[consentType]) {
+      return defaultConsentBlock[consentType] || UNSPECIFIED_CONSENT;
+    }
+    if (GOOGLE_ALLOWED_CONSENT_STATUS.includes(consents[consentType])) {
+      return consents[consentType];
+    }
+    return defaultConsentBlock[consentType] || UNKNOWN_CONSENT;
+  };
+
+  // Construct consentObj based on the common consent fields
+  const consentObj = ADWORDS_CONSENT_FILEDS.reduce((obj, consentType) => {
+    // eslint-disable-next-line no-param-reassign
+    obj[consentType] = processConsent(consentType);
+    return obj;
+  }, {});
+
+  return consentObj;
+};
+
+module.exports = {
+  populateConsentFromConfig,
+  UNSPECIFIED_CONSENT,
+  UNKNOWN_CONSENT,
+  GOOGLE_ALLOWED_CONSENT_STATUS,
+  finaliseConsent,
+};