fix: standardise hashing for all CAPI integrations (#3379)

* fix: standardize hashing for all CAPI integrations * fix: use trim before hashing for GAOC,GAEC,impact,YDSP,fb and fbconv
rudderlabs · May 22, 2024 · c249a69 · c249a69
1 parent 90e60a5
commit c249a69
Show file tree

Hide file tree

Showing 17 changed files with 46 additions and 42 deletions.
diff --git a/src/cdk/v2/destinations/reddit/procWorkflow.yaml b/src/cdk/v2/destinations/reddit/procWorkflow.yaml
@@ -36,13 +36,13 @@ steps:
           const os = (.message.context.os.name)?  .message.context.os.name.toLowerCase(): null;
           const hashData = .destination.Config.hashData;
           let user = .message.().({
-            "email": hashData ? $.SHA256({{{{$.getGenericPaths("email")}}}}) : ({{{{$.getGenericPaths("email")}}}}),  
-            "external_id": hashData ? $.SHA256({{{{$.getGenericPaths("userId")}}}}) : ({{{{$.getGenericPaths("userId")}}}}),  
-            "ip_address": hashData? $.SHA256(.context.ip || .request_ip) : (.context.ip || .request_ip),
+            "email": hashData ? $.SHA256({{{{$.getGenericPaths("email")}}}}.trim()) : ({{{{$.getGenericPaths("email")}}}}),  
+            "external_id": hashData ? $.SHA256({{{{$.getGenericPaths("userId")}}}}.trim()) : ({{{{$.getGenericPaths("userId")}}}}),  
+            "ip_address": hashData? $.SHA256(.context.ip.trim() || .request_ip.trim()) : (.context.ip || .request_ip),
             "uuid": .properties.uuid,
             "user_agent": .context.userAgent,
-            "idfa": $.isAppleFamily(os)? (hashData? $.SHA256(.context.device.advertisingId): .context.device.advertisingId): null,
-            "aaid": os === "android" && .context.device ? (hashData? $.SHA256(.context.device.advertisingId): .context.device.advertisingId): null,
+            "idfa": $.isAppleFamily(os)? (hashData? $.SHA256(.context.device.advertisingId.trim()): .context.device.advertisingId): null,
+            "aaid": os === "android" && .context.device ? (hashData? $.SHA256(.context.device.advertisingId.trim()): .context.device.advertisingId): null,
             "opt_out": .properties.optOut,
             "screen_dimensions": {"width": .context.screen.width, "height":  .context.screen.height},
           });

diff --git a/src/v0/destinations/facebook_offline_conversions/utils.js b/src/v0/destinations/facebook_offline_conversions/utils.js
@@ -396,7 +396,7 @@ const preparePayload = (facebookOfflineConversionsPayload, destination) => {
   const keys = Object.keys(facebookOfflineConversionsPayload);
   keys.forEach((key) => {
     if (isHashRequired && HASHING_REQUIRED_KEYS.includes(key)) {
-      payload[key] = sha256(facebookOfflineConversionsPayload[key]);
+      payload[key] = sha256(facebookOfflineConversionsPayload[key].trim());
     } else {
       payload[key] = facebookOfflineConversionsPayload[key];
     }
@@ -407,8 +407,8 @@ const preparePayload = (facebookOfflineConversionsPayload, destination) => {
       ? facebookOfflineConversionsPayload.name.split(' ')
       : null;
     if (split !== null && Array.isArray(split) && split.length === 2) {
-      payload.fn = isHashRequired ? sha256(split[0]) : split[0];
-      payload.ln = isHashRequired ? sha256(split[1]) : split[1];
+      payload.fn = isHashRequired ? sha256(split[0].trim()) : split[0];
+      payload.ln = isHashRequired ? sha256(split[1].trim()) : split[1];
     }
     delete payload.name;
   }

diff --git a/src/v0/destinations/fb/transform.js b/src/v0/destinations/fb/transform.js
@@ -90,7 +90,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) {
           clonedUpdatedEvent[prop] !== ''
         ) {
           isUDSet = true;
-          clonedUpdatedEvent[prop] = sha256(clonedUpdatedEvent[prop].toLowerCase());
+          clonedUpdatedEvent[prop] = sha256(clonedUpdatedEvent[prop].trim().toLowerCase());
         }
         break;
       case 'ud[zp]':
@@ -113,7 +113,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) {
           } else {
             isUDSet = true;
             clonedUpdatedEvent[prop] = sha256(
-              clonedUpdatedEvent[prop].toLowerCase() === 'female' ? 'f' : 'm',
+              clonedUpdatedEvent[prop].trim().toLowerCase() === 'female' ? 'f' : 'm',
             );
           }
         }
@@ -128,7 +128,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) {
         if (clonedUpdatedEvent[prop] && clonedUpdatedEvent[prop] !== '') {
           isUDSet = true;
           clonedUpdatedEvent[prop] = sha256(
-            clonedUpdatedEvent[prop].toLowerCase().replace(/ /g, ''),
+            clonedUpdatedEvent[prop].trim().toLowerCase().replace(/ /g, ''),
           );
         }
         break;

diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json b/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json
@@ -55,7 +55,7 @@
     "sourceFromGenericMap": true,
     "required": false,
     "metadata": {
-      "type": "hashToSha256"
+      "type": ["trim", "hashToSha256"]
     }
   },
   {
@@ -64,7 +64,7 @@
     "sourceFromGenericMap": true,
     "required": false,
     "metadata": {
-      "type": "hashToSha256"
+      "type": ["trim", "hashToSha256"]
     }
   },
   {
@@ -73,7 +73,7 @@
     "sourceFromGenericMap": true,
     "required": false,
     "metadata": {
-      "type": "hashToSha256"
+      "type": ["trim", "hashToSha256"]
     }
   },
   {
@@ -82,7 +82,7 @@
     "sourceFromGenericMap": true,
     "required": false,
     "metadata": {
-      "type": "hashToSha256"
+      "type": ["trim", "hashToSha256"]
     }
   },
   {
@@ -127,7 +127,7 @@
     "sourceKeys": ["context.traits.streetAddress", "context.traits.address"],
     "required": false,
     "metadata": {
-      "type": "hashToSha256"
+      "type": ["trim", "hashToSha256"]
     }
   }
 ]
diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js
@@ -24,7 +24,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant');
 const updateMappingJson = (mapping) => {
   const newMapping = [];
   mapping.forEach((element) => {
-    if (get(element, 'metadata.type') && element.metadata.type === 'hashToSha256') {
+    if (get(element, 'metadata.type') && element.metadata.type.includes('hashToSha256')) {
       element.metadata.type = 'toString';
     }
     newMapping.push(element);

diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js
@@ -140,17 +140,17 @@ const buildAndGetAddress = (message, hashUserIdentifier) => {
   const address = constructPayload(message, trackAddStoreAddressConversionsMapping);
   if (address.hashed_last_name) {
     address.hashed_last_name = hashUserIdentifier
-      ? sha256(address.hashed_last_name).toString()
+      ? sha256(address.hashed_last_name.trim()).toString()
       : address.hashed_last_name;
   }
   if (address.hashed_first_name) {
     address.hashed_first_name = hashUserIdentifier
-      ? sha256(address.hashed_first_name).toString()
+      ? sha256(address.hashed_first_name.trim()).toString()
       : address.hashed_first_name;
   }
   if (address.hashed_street_address) {
     address.hashed_street_address = hashUserIdentifier
-      ? sha256(address.hashed_street_address).toString()
+      ? sha256(address.hashed_street_address.trim()).toString()
       : address.hashed_street_address;
   }
   return Object.keys(address).length > 0 ? address : null;
@@ -269,8 +269,10 @@ const getAddConversionPayload = (message, Config) => {
   const phone = getFieldValueFromMessage(message, 'phone');
 
   const userIdentifierInfo = {
-    email: hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email).toString() : email,
-    phone: hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone).toString() : phone,
+    email:
+      hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email.trim()).toString() : email,
+    phone:
+      hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone.trim()).toString() : phone,
     address: buildAndGetAddress(message, hashUserIdentifier),
   };
 
@@ -363,8 +365,10 @@ const getClickConversionPayloadAndEndpoint = (
   // Ref - https://developers.google.com/google-ads/api/rest/reference/rest/v11/customers/uploadClickConversions#ClickConversion
 
   const userIdentifierInfo = {
-    email: hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email).toString() : email,
-    phone: hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone).toString() : phone,
+    email:
+      hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email.trim()).toString() : email,
+    phone:
+      hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone.trim()).toString() : phone,
   };
 
   const keyName = getExisitingUserIdentifier(userIdentifierInfo, defaultUserIdentifier);

diff --git a/src/v0/destinations/impact/transform.js b/src/v0/destinations/impact/transform.js
@@ -59,7 +59,7 @@ const buildPageLoadPayload = (message, campaignId, impactAppId, enableEmailHashi
   let payload = constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.PAGELOAD.name]);
   if (isDefinedAndNotNull(payload.CustomerEmail)) {
     payload.CustomerEmail = enableEmailHashing
-      ? sha1(payload?.CustomerEmail)
+      ? sha1(payload?.CustomerEmail.trim())
       : payload?.CustomerEmail;
   }
   payload.CampaignId = campaignId;
@@ -155,7 +155,7 @@ const processTrackEvent = (message, Config) => {
     payload.ImpactAppId = impactAppId;
     if (isDefinedAndNotNull(payload.CustomerEmail)) {
       payload.CustomerEmail = enableEmailHashing
-        ? sha1(payload?.CustomerEmail)
+        ? sha1(payload?.CustomerEmail.trim())
         : payload?.CustomerEmail;
     }
 

diff --git a/src/v0/destinations/pinterest_tag/utils.js b/src/v0/destinations/pinterest_tag/utils.js
@@ -41,8 +41,8 @@ const getHashedValue = (key, value) => {
     case 'fn':
     case 'ge':
       value = Array.isArray(value)
-        ? value.map((val) => val.toString().toLowerCase())
-        : value.toString().toLowerCase();
+        ? value.map((val) => val.toString().trim().toLowerCase())
+        : value.toString().trim().toLowerCase();
       break;
     case 'ph':
       // phone numbers should only contain digits & should not contain leading zeros
@@ -53,7 +53,7 @@ const getHashedValue = (key, value) => {
     case 'zp':
       // zip fields should only contain digits
       value = Array.isArray(value)
-        ? value.map((val) => val.toString().replace(/\D/g, ''))
+        ? value.map((val) => val.toString().trim().replace(/\D/g, ''))
         : value.toString().replace(/\D/g, '');
       break;
     case 'hashed_maids':

diff --git a/src/v0/destinations/yahoo_dsp/util.js b/src/v0/destinations/yahoo_dsp/util.js
@@ -51,7 +51,7 @@ const populateIdentifiers = (audienceList, Config) => {
       }
       // here, hashing the data if is not hashed and pushing in the seedList array.
       if (hashRequired) {
-        seedList.push(sha256(userTraits[audienceAttribute]));
+        seedList.push(sha256(userTraits[audienceAttribute].trim()));
       } else {
         seedList.push(userTraits[audienceAttribute]);
       }

diff --git a/test/integrations/destinations/facebook_offline_conversions/processor/data.ts b/test/integrations/destinations/facebook_offline_conversions/processor/data.ts
@@ -1436,7 +1436,7 @@ export const data = [
                 traits: {
                   email: '[email protected]',
                   birthday: '2005-01-01T23:28:56.782Z',
-                  firstName: 'test',
+                  firstName: ' test',
                   name: 'test rudderlabs',
                   address: {
                     city: 'kalkata',

diff --git a/test/integrations/destinations/fb/processor/data.ts b/test/integrations/destinations/fb/processor/data.ts
@@ -497,7 +497,7 @@ export const data = [
                 traits: {
                   email: '[email protected]',
                   anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1',
-                  firstName: 'test',
+                  firstName: ' test',
                   lastName: 'last',
                   gender: 1234,
                   phone: '+91-9831311135',

diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts
@@ -1255,7 +1255,7 @@ export const data = [
                 },
                 traits: {
                   phone: '912382193',
-                  firstName: 'John',
+                  firstName: ' John',
                   lastName: 'Gomes',
                   city: 'London',
                   state: 'UK',

diff --git a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts
@@ -321,7 +321,7 @@ export const data = [
                   advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8',
                 },
                 traits: {
-                  email: '[email protected]',
+                  email: ' [email protected]',
                   phone: '+1-202-555-0146',
                   firstName: 'John',
                   lastName: 'Gomes',

diff --git a/test/integrations/destinations/impact/processor/data.ts b/test/integrations/destinations/impact/processor/data.ts
@@ -26,7 +26,7 @@ export const data = [
                   namespace: 'com.rudderlabs.javascript',
                 },
                 traits: {
-                  email: '[email protected]',
+                  email: ' [email protected]',
                   phone: '+917836362334',
                   userId: 'user123',
                 },

diff --git a/test/integrations/destinations/pinterest_tag/processor/data.ts b/test/integrations/destinations/pinterest_tag/processor/data.ts
@@ -19,8 +19,8 @@ export const data = [
                 userAgent: 'chrome',
                 traits: {
                   anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1',
-                  email: '[email protected]',
-                  phone: '+1234589947',
+                  email: ' [email protected]',
+                  phone: '+1234589947 ',
                   gender: 'non-binary',
                   db: '19950715',
                   lastname: 'Rudderlabs',

diff --git a/test/integrations/destinations/reddit/processor/data.ts b/test/integrations/destinations/reddit/processor/data.ts
@@ -12,13 +12,13 @@ export const data = [
             message: {
               context: {
                 traits: {
-                  email: '[email protected]',
+                  email: '[email protected] ',
                 },
                 userAgent:
                   'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36',
-                ip: '54.100.200.255',
+                ip: ' 54.100.200.255',
                 device: {
-                  advertisingId: 'asfds7fdsihf734b34j43f',
+                  advertisingId: ' asfds7fdsihf734b34j43f',
                 },
                 os: {
                   name: 'android',
@@ -29,7 +29,7 @@ export const data = [
               originalTimestamp: '2019-10-14T09:03:17.562Z',
               anonymousId: '123456',
               event: 'Order Completed',
-              userId: 'testuserId1',
+              userId: ' testuserId1',
               properties: {
                 checkout_id: '12345',
                 order_id: '1234',

diff --git a/test/integrations/destinations/yahoo_dsp/processor/data.ts b/test/integrations/destinations/yahoo_dsp/processor/data.ts
@@ -51,7 +51,7 @@ export const data = [
                     },
                     {
                       ipAddress: 'fdffddf',
-                      email: '[email protected]',
+                      email: '[email protected] ',
                       deviceId: 'djfdjfkdjf',
                       phone: '@09876543210',
                       firstName: 'test',