From 89555119df72cc1cd997ad51d6b08d83c22ff128 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 16 May 2024 17:38:22 +0530 Subject: [PATCH 1/2] fix: standardize hashing for all CAPI integrations --- src/cdk/v2/destinations/reddit/procWorkflow.yaml | 10 +++++----- src/v0/destinations/pinterest_tag/utils.js | 6 +++--- .../destinations/pinterest_tag/processor/data.ts | 4 ++-- .../integrations/destinations/reddit/processor/data.ts | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/cdk/v2/destinations/reddit/procWorkflow.yaml b/src/cdk/v2/destinations/reddit/procWorkflow.yaml index 7b989f15e4..06d2c95f25 100644 --- a/src/cdk/v2/destinations/reddit/procWorkflow.yaml +++ b/src/cdk/v2/destinations/reddit/procWorkflow.yaml @@ -36,13 +36,13 @@ steps: const os = (.message.context.os.name)? .message.context.os.name.toLowerCase(): null; const hashData = .destination.Config.hashData; let user = .message.().({ - "email": hashData ? $.SHA256({{{{$.getGenericPaths("email")}}}}) : ({{{{$.getGenericPaths("email")}}}}), - "external_id": hashData ? $.SHA256({{{{$.getGenericPaths("userId")}}}}) : ({{{{$.getGenericPaths("userId")}}}}), - "ip_address": hashData? $.SHA256(.context.ip || .request_ip) : (.context.ip || .request_ip), + "email": hashData ? $.SHA256({{{{$.getGenericPaths("email")}}}}.trim()) : ({{{{$.getGenericPaths("email")}}}}), + "external_id": hashData ? $.SHA256({{{{$.getGenericPaths("userId")}}}}.trim()) : ({{{{$.getGenericPaths("userId")}}}}), + "ip_address": hashData? $.SHA256(.context.ip.trim() || .request_ip.trim()) : (.context.ip || .request_ip), "uuid": .properties.uuid, "user_agent": .context.userAgent, - "idfa": $.isAppleFamily(os)? (hashData? $.SHA256(.context.device.advertisingId): .context.device.advertisingId): null, - "aaid": os === "android" && .context.device ? (hashData? $.SHA256(.context.device.advertisingId): .context.device.advertisingId): null, + "idfa": $.isAppleFamily(os)? (hashData? $.SHA256(.context.device.advertisingId.trim()): .context.device.advertisingId): null, + "aaid": os === "android" && .context.device ? (hashData? $.SHA256(.context.device.advertisingId.trim()): .context.device.advertisingId): null, "opt_out": .properties.optOut, "screen_dimensions": {"width": .context.screen.width, "height": .context.screen.height}, }); diff --git a/src/v0/destinations/pinterest_tag/utils.js b/src/v0/destinations/pinterest_tag/utils.js index 340fba498e..57d595571f 100644 --- a/src/v0/destinations/pinterest_tag/utils.js +++ b/src/v0/destinations/pinterest_tag/utils.js @@ -41,8 +41,8 @@ const getHashedValue = (key, value) => { case 'fn': case 'ge': value = Array.isArray(value) - ? value.map((val) => val.toString().toLowerCase()) - : value.toString().toLowerCase(); + ? value.map((val) => val.toString().trim().toLowerCase()) + : value.toString().trim().toLowerCase(); break; case 'ph': // phone numbers should only contain digits & should not contain leading zeros @@ -53,7 +53,7 @@ const getHashedValue = (key, value) => { case 'zp': // zip fields should only contain digits value = Array.isArray(value) - ? value.map((val) => val.toString().replace(/\D/g, '')) + ? value.map((val) => val.toString().trim().replace(/\D/g, '')) : value.toString().replace(/\D/g, ''); break; case 'hashed_maids': diff --git a/test/integrations/destinations/pinterest_tag/processor/data.ts b/test/integrations/destinations/pinterest_tag/processor/data.ts index 65b33e7740..48b624645f 100644 --- a/test/integrations/destinations/pinterest_tag/processor/data.ts +++ b/test/integrations/destinations/pinterest_tag/processor/data.ts @@ -19,8 +19,8 @@ export const data = [ userAgent: 'chrome', traits: { anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', - email: 'abc@gmail.com', - phone: '+1234589947', + email: ' abc@gmail.com', + phone: '+1234589947 ', gender: 'non-binary', db: '19950715', lastname: 'Rudderlabs', diff --git a/test/integrations/destinations/reddit/processor/data.ts b/test/integrations/destinations/reddit/processor/data.ts index a97ae23d2a..e7b36f56ff 100644 --- a/test/integrations/destinations/reddit/processor/data.ts +++ b/test/integrations/destinations/reddit/processor/data.ts @@ -12,13 +12,13 @@ export const data = [ message: { context: { traits: { - email: 'testone@gmail.com', + email: 'testone@gmail.com ', }, userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - ip: '54.100.200.255', + ip: ' 54.100.200.255', device: { - advertisingId: 'asfds7fdsihf734b34j43f', + advertisingId: ' asfds7fdsihf734b34j43f', }, os: { name: 'android', @@ -29,7 +29,7 @@ export const data = [ originalTimestamp: '2019-10-14T09:03:17.562Z', anonymousId: '123456', event: 'Order Completed', - userId: 'testuserId1', + userId: ' testuserId1', properties: { checkout_id: '12345', order_id: '1234', From 0c574d7cda4aeb71f40f7b2f93817edba1a4f0c2 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 21 May 2024 12:53:43 +0530 Subject: [PATCH 2/2] fix: use trim before hashing for GAOC,GAEC,impact,YDSP,fb and fbconv --- .../facebook_offline_conversions/utils.js | 6 +++--- src/v0/destinations/fb/transform.js | 6 +++--- .../data/trackConfig.json | 10 +++++----- .../transform.js | 2 +- .../utils.js | 18 +++++++++++------- src/v0/destinations/impact/transform.js | 4 ++-- src/v0/destinations/yahoo_dsp/util.js | 2 +- .../processor/data.ts | 2 +- .../destinations/fb/processor/data.ts | 2 +- .../processor/data.ts | 2 +- .../processor/data.ts | 2 +- .../destinations/impact/processor/data.ts | 2 +- .../destinations/yahoo_dsp/processor/data.ts | 2 +- 13 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/v0/destinations/facebook_offline_conversions/utils.js b/src/v0/destinations/facebook_offline_conversions/utils.js index c48de4e0b9..460ef71176 100644 --- a/src/v0/destinations/facebook_offline_conversions/utils.js +++ b/src/v0/destinations/facebook_offline_conversions/utils.js @@ -396,7 +396,7 @@ const preparePayload = (facebookOfflineConversionsPayload, destination) => { const keys = Object.keys(facebookOfflineConversionsPayload); keys.forEach((key) => { if (isHashRequired && HASHING_REQUIRED_KEYS.includes(key)) { - payload[key] = sha256(facebookOfflineConversionsPayload[key]); + payload[key] = sha256(facebookOfflineConversionsPayload[key].trim()); } else { payload[key] = facebookOfflineConversionsPayload[key]; } @@ -407,8 +407,8 @@ const preparePayload = (facebookOfflineConversionsPayload, destination) => { ? facebookOfflineConversionsPayload.name.split(' ') : null; if (split !== null && Array.isArray(split) && split.length === 2) { - payload.fn = isHashRequired ? sha256(split[0]) : split[0]; - payload.ln = isHashRequired ? sha256(split[1]) : split[1]; + payload.fn = isHashRequired ? sha256(split[0].trim()) : split[0]; + payload.ln = isHashRequired ? sha256(split[1].trim()) : split[1]; } delete payload.name; } diff --git a/src/v0/destinations/fb/transform.js b/src/v0/destinations/fb/transform.js index e6f8e986cf..1160cef407 100644 --- a/src/v0/destinations/fb/transform.js +++ b/src/v0/destinations/fb/transform.js @@ -90,7 +90,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { clonedUpdatedEvent[prop] !== '' ) { isUDSet = true; - clonedUpdatedEvent[prop] = sha256(clonedUpdatedEvent[prop].toLowerCase()); + clonedUpdatedEvent[prop] = sha256(clonedUpdatedEvent[prop].trim().toLowerCase()); } break; case 'ud[zp]': @@ -113,7 +113,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { } else { isUDSet = true; clonedUpdatedEvent[prop] = sha256( - clonedUpdatedEvent[prop].toLowerCase() === 'female' ? 'f' : 'm', + clonedUpdatedEvent[prop].trim().toLowerCase() === 'female' ? 'f' : 'm', ); } } @@ -128,7 +128,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { if (clonedUpdatedEvent[prop] && clonedUpdatedEvent[prop] !== '') { isUDSet = true; clonedUpdatedEvent[prop] = sha256( - clonedUpdatedEvent[prop].toLowerCase().replace(/ /g, ''), + clonedUpdatedEvent[prop].trim().toLowerCase().replace(/ /g, ''), ); } break; diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json b/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json index bf5485270b..c38b24598d 100644 --- a/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json +++ b/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json @@ -55,7 +55,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -64,7 +64,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -73,7 +73,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -82,7 +82,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -127,7 +127,7 @@ "sourceKeys": ["context.traits.streetAddress", "context.traits.address"], "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } } ] diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js index 0be7c3f0ee..55d0c16c8c 100644 --- a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js +++ b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js @@ -24,7 +24,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); const updateMappingJson = (mapping) => { const newMapping = []; mapping.forEach((element) => { - if (get(element, 'metadata.type') && element.metadata.type === 'hashToSha256') { + if (get(element, 'metadata.type') && element.metadata.type.includes('hashToSha256')) { element.metadata.type = 'toString'; } newMapping.push(element); diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js index 70b42e2157..dfa892a769 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.js @@ -140,17 +140,17 @@ const buildAndGetAddress = (message, hashUserIdentifier) => { const address = constructPayload(message, trackAddStoreAddressConversionsMapping); if (address.hashed_last_name) { address.hashed_last_name = hashUserIdentifier - ? sha256(address.hashed_last_name).toString() + ? sha256(address.hashed_last_name.trim()).toString() : address.hashed_last_name; } if (address.hashed_first_name) { address.hashed_first_name = hashUserIdentifier - ? sha256(address.hashed_first_name).toString() + ? sha256(address.hashed_first_name.trim()).toString() : address.hashed_first_name; } if (address.hashed_street_address) { address.hashed_street_address = hashUserIdentifier - ? sha256(address.hashed_street_address).toString() + ? sha256(address.hashed_street_address.trim()).toString() : address.hashed_street_address; } return Object.keys(address).length > 0 ? address : null; @@ -269,8 +269,10 @@ const getAddConversionPayload = (message, Config) => { const phone = getFieldValueFromMessage(message, 'phone'); const userIdentifierInfo = { - email: hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email).toString() : email, - phone: hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone).toString() : phone, + email: + hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email.trim()).toString() : email, + phone: + hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone.trim()).toString() : phone, address: buildAndGetAddress(message, hashUserIdentifier), }; @@ -363,8 +365,10 @@ const getClickConversionPayloadAndEndpoint = ( // Ref - https://developers.google.com/google-ads/api/rest/reference/rest/v11/customers/uploadClickConversions#ClickConversion const userIdentifierInfo = { - email: hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email).toString() : email, - phone: hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone).toString() : phone, + email: + hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email.trim()).toString() : email, + phone: + hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone.trim()).toString() : phone, }; const keyName = getExisitingUserIdentifier(userIdentifierInfo, defaultUserIdentifier); diff --git a/src/v0/destinations/impact/transform.js b/src/v0/destinations/impact/transform.js index 2eefdf7992..729f988938 100644 --- a/src/v0/destinations/impact/transform.js +++ b/src/v0/destinations/impact/transform.js @@ -59,7 +59,7 @@ const buildPageLoadPayload = (message, campaignId, impactAppId, enableEmailHashi let payload = constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.PAGELOAD.name]); if (isDefinedAndNotNull(payload.CustomerEmail)) { payload.CustomerEmail = enableEmailHashing - ? sha1(payload?.CustomerEmail) + ? sha1(payload?.CustomerEmail.trim()) : payload?.CustomerEmail; } payload.CampaignId = campaignId; @@ -155,7 +155,7 @@ const processTrackEvent = (message, Config) => { payload.ImpactAppId = impactAppId; if (isDefinedAndNotNull(payload.CustomerEmail)) { payload.CustomerEmail = enableEmailHashing - ? sha1(payload?.CustomerEmail) + ? sha1(payload?.CustomerEmail.trim()) : payload?.CustomerEmail; } diff --git a/src/v0/destinations/yahoo_dsp/util.js b/src/v0/destinations/yahoo_dsp/util.js index 255f84d1c9..54002a3bce 100644 --- a/src/v0/destinations/yahoo_dsp/util.js +++ b/src/v0/destinations/yahoo_dsp/util.js @@ -51,7 +51,7 @@ const populateIdentifiers = (audienceList, Config) => { } // here, hashing the data if is not hashed and pushing in the seedList array. if (hashRequired) { - seedList.push(sha256(userTraits[audienceAttribute])); + seedList.push(sha256(userTraits[audienceAttribute].trim())); } else { seedList.push(userTraits[audienceAttribute]); } diff --git a/test/integrations/destinations/facebook_offline_conversions/processor/data.ts b/test/integrations/destinations/facebook_offline_conversions/processor/data.ts index 26d9a5e2f9..90709b67a2 100644 --- a/test/integrations/destinations/facebook_offline_conversions/processor/data.ts +++ b/test/integrations/destinations/facebook_offline_conversions/processor/data.ts @@ -1436,7 +1436,7 @@ export const data = [ traits: { email: 'test@rudderstack.com', birthday: '2005-01-01T23:28:56.782Z', - firstName: 'test', + firstName: ' test', name: 'test rudderlabs', address: { city: 'kalkata', diff --git a/test/integrations/destinations/fb/processor/data.ts b/test/integrations/destinations/fb/processor/data.ts index a437b90855..b0b4ba9ecf 100644 --- a/test/integrations/destinations/fb/processor/data.ts +++ b/test/integrations/destinations/fb/processor/data.ts @@ -497,7 +497,7 @@ export const data = [ traits: { email: 'abc@gmail.com', anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', - firstName: 'test', + firstName: ' test', lastName: 'last', gender: 1234, phone: '+91-9831311135', diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts index 0a9542a5d5..13b2609bf8 100644 --- a/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts +++ b/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts @@ -1255,7 +1255,7 @@ export const data = [ }, traits: { phone: '912382193', - firstName: 'John', + firstName: ' John', lastName: 'Gomes', city: 'London', state: 'UK', diff --git a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts index decb1e58c7..ab3e19dc2f 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts @@ -321,7 +321,7 @@ export const data = [ advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8', }, traits: { - email: 'alex@example.com', + email: ' alex@example.com', phone: '+1-202-555-0146', firstName: 'John', lastName: 'Gomes', diff --git a/test/integrations/destinations/impact/processor/data.ts b/test/integrations/destinations/impact/processor/data.ts index 1e4e91e7ad..0841f44c62 100644 --- a/test/integrations/destinations/impact/processor/data.ts +++ b/test/integrations/destinations/impact/processor/data.ts @@ -26,7 +26,7 @@ export const data = [ namespace: 'com.rudderlabs.javascript', }, traits: { - email: 'user123@email.com', + email: ' user123@email.com', phone: '+917836362334', userId: 'user123', }, diff --git a/test/integrations/destinations/yahoo_dsp/processor/data.ts b/test/integrations/destinations/yahoo_dsp/processor/data.ts index eb607d60fb..3d04f9aa5c 100644 --- a/test/integrations/destinations/yahoo_dsp/processor/data.ts +++ b/test/integrations/destinations/yahoo_dsp/processor/data.ts @@ -51,7 +51,7 @@ export const data = [ }, { ipAddress: 'fdffddf', - email: 'van@abc.com', + email: 'van@abc.com ', deviceId: 'djfdjfkdjf', phone: '@09876543210', firstName: 'test',