From 6a99c8971e954327ef74bcb5bf3fc1e9b9d055d6 Mon Sep 17 00:00:00 2001 From: Dilip Kola Date: Thu, 13 Jun 2024 17:47:35 +0530 Subject: [PATCH 1/4] chore: upgrade json template packages --- package-lock.json | 32 ++++++++++++++++---------------- package.json | 4 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6a35944fb4..04ee1c5a38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,8 +20,8 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.8", - "@rudderstack/json-template-engine": "^0.11.0", - "@rudderstack/workflow-engine": "^0.8.0", + "@rudderstack/json-template-engine": "^0.13.0", + "@rudderstack/workflow-engine": "^0.8.1", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", @@ -4482,17 +4482,17 @@ } }, "node_modules/@rudderstack/json-template-engine": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.11.0.tgz", - "integrity": "sha512-9XrzY7W9mL2lYro2NOSInuDElW7Qk0nP61UbrfJiTQfrzbyaH7ml663eD07a/4ia3uQynITPsSIGHpMgP3qlEw==" + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.13.0.tgz", + "integrity": "sha512-nROhLP8Q8wYdeb7N1qpJvkJRI1b7J9jUFxCcBqeN8C/j1kHTAHgomilQjOnGFb1H+OP6IxI93b+JJ24mIDHXnw==" }, "node_modules/@rudderstack/workflow-engine": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.8.0.tgz", - "integrity": "sha512-oBRucBNR29E2PzwHX3hANT0c6V0yFKNMWxDg0jr8Hin4co6KZjxi4FdpkzTNWvk2h+8iXT8NCSPdJBjt03hTrw==", + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.8.1.tgz", + "integrity": "sha512-6oBG6lLljwci8C9MFqoKyJf+MTwsOyUMWg9yQD5/49nbNdTvazXQ7zpvjzEGgTxe7Ub1ppoDe5eSqZYSoWIAvg==", "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", - "@rudderstack/json-template-engine": "^0.11.0", + "@rudderstack/json-template-engine": "^0.13.0", "jsonata": "^2.0.5", "lodash": "^4.17.21", "object-sizeof": "^2.6.4", @@ -6559,11 +6559,11 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -10391,9 +10391,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dependencies": { "to-regex-range": "^5.0.1" }, diff --git a/package.json b/package.json index 2acfed56ff..12ea084bf4 100644 --- a/package.json +++ b/package.json @@ -65,8 +65,8 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.8", - "@rudderstack/json-template-engine": "^0.11.0", - "@rudderstack/workflow-engine": "^0.8.0", + "@rudderstack/json-template-engine": "^0.13.0", + "@rudderstack/workflow-engine": "^0.8.1", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", From 3aee494b3902f6e598bc9fb8d1f165b637cfcf02 Mon Sep 17 00:00:00 2001 From: Dilip Kola Date: Fri, 14 Jun 2024 10:39:28 +0530 Subject: [PATCH 2/4] chore: add verify server start workflow --- .github/workflows/verify-server-start.yml | 34 +++++++++++++++++++++++ package-lock.json | 18 ++++++------ package.json | 4 +-- 3 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/verify-server-start.yml diff --git a/.github/workflows/verify-server-start.yml b/.github/workflows/verify-server-start.yml new file mode 100644 index 0000000000..6ac5b7be05 --- /dev/null +++ b/.github/workflows/verify-server-start.yml @@ -0,0 +1,34 @@ +name: Verify Server start + +on: + pull_request: + types: ['opened', 'reopened', 'synchronize'] + +jobs: + check-health: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 1 + + - name: Setup Node + uses: actions/setup-node@v4.0.2 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - name: Install Dependencies + run: npm ci + + - name: Start server + run: npm run build:start & + + - name: Wait for server to start + run: sleep 10 # Adjust the time as necessary for your server to start + + - name: Check server health + run: | + curl --fail http://localhost:9090/health || exit 1 diff --git a/package-lock.json b/package-lock.json index 04ee1c5a38..3166828d24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,8 +20,8 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.8", - "@rudderstack/json-template-engine": "^0.13.0", - "@rudderstack/workflow-engine": "^0.8.1", + "@rudderstack/json-template-engine": "^0.13.2", + "@rudderstack/workflow-engine": "^0.8.2", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", @@ -4482,17 +4482,17 @@ } }, "node_modules/@rudderstack/json-template-engine": { - "version": "0.13.0", - "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.13.0.tgz", - "integrity": "sha512-nROhLP8Q8wYdeb7N1qpJvkJRI1b7J9jUFxCcBqeN8C/j1kHTAHgomilQjOnGFb1H+OP6IxI93b+JJ24mIDHXnw==" + "version": "0.13.2", + "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.13.2.tgz", + "integrity": "sha512-uEyMv/qjm/mP5V8EifJzolvFLtka/dacmvwo9Xk3+MnEbsNN0YLu7Z/qWeyXeDF5chvy8JfaqV8lNgO3SxVG7g==" }, "node_modules/@rudderstack/workflow-engine": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.8.1.tgz", - "integrity": "sha512-6oBG6lLljwci8C9MFqoKyJf+MTwsOyUMWg9yQD5/49nbNdTvazXQ7zpvjzEGgTxe7Ub1ppoDe5eSqZYSoWIAvg==", + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.8.2.tgz", + "integrity": "sha512-cjn3J8CUarAE3cbASRvkf7A2745Clzkw/ffqGLzD8+9KvTN6mC28Pm9c5169LPDmt+NMUMw0W5xHgNO3cV9eqQ==", "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", - "@rudderstack/json-template-engine": "^0.13.0", + "@rudderstack/json-template-engine": "^0.13.2", "jsonata": "^2.0.5", "lodash": "^4.17.21", "object-sizeof": "^2.6.4", diff --git a/package.json b/package.json index 12ea084bf4..855c7ae3a3 100644 --- a/package.json +++ b/package.json @@ -65,8 +65,8 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.8", - "@rudderstack/json-template-engine": "^0.13.0", - "@rudderstack/workflow-engine": "^0.8.1", + "@rudderstack/json-template-engine": "^0.13.2", + "@rudderstack/workflow-engine": "^0.8.2", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", From d4d5a89951414f02d02101a6582b6e9fa8f9a7a5 Mon Sep 17 00:00:00 2001 From: Paul K <59660521+zenpaul@users.noreply.github.com> Date: Fri, 14 Jun 2024 03:21:24 -0500 Subject: [PATCH 3/4] feat(integrations/auth0): add Auth0 event type to event context (#3433) * feat(integrations/auth0): add Auth0 event type to event context * feat(integrations/auth0): move Auth0 event type to event properties * feat(integrations): better --- src/v0/sources/auth0/mapping.json | 2 +- test/integrations/sources/auth0/data.ts | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/v0/sources/auth0/mapping.json b/src/v0/sources/auth0/mapping.json index bc5869a19b..dcbd389945 100644 --- a/src/v0/sources/auth0/mapping.json +++ b/src/v0/sources/auth0/mapping.json @@ -65,6 +65,6 @@ }, { "sourceKeys": "type", - "destKeys": "source_type" + "destKeys": "properties.source_type" } ] diff --git a/test/integrations/sources/auth0/data.ts b/test/integrations/sources/auth0/data.ts index b115f0e05e..44b511cad2 100644 --- a/test/integrations/sources/auth0/data.ts +++ b/test/integrations/sources/auth0/data.ts @@ -254,7 +254,6 @@ export const data = [ batch: [ { type: 'identify', - source_type: 'ss', sentAt: '2022-10-31T05:57:06.859Z', traits: { connection: 'Username-Password-Authentication', @@ -291,6 +290,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: 'All Applications', description: '', + source_type: 'ss', }, integrations: { Auth0: false, @@ -305,7 +305,6 @@ export const data = [ batch: [ { type: 'track', - source_type: 'sapi', event: 'Success API Operation', sentAt: '2022-10-31T05:57:06.874Z', userId: 'auth0|dummyPassword', @@ -511,6 +510,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: '', description: 'Create a User', + source_type: 'sapi', }, integrations: { Auth0: false, @@ -596,7 +596,6 @@ export const data = [ batch: [ { type: 'group', - source_type: 'sapi', sentAt: '2022-10-31T06:09:59.135Z', userId: 'google-oauth2|123456', anonymousId: '97fcd7b2-cc24-47d7-b776-057b7b199513', @@ -650,6 +649,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: '', description: 'Add members to an organization', + source_type: 'sapi', }, integrations: { Auth0: false, @@ -939,7 +939,6 @@ export const data = [ batch: [ { type: 'track', - source_type: 'sapi', event: 'Success API Operation', sentAt: '2022-10-31T06:15:25.201Z', userId: 'google-oauth2|123456', @@ -1147,6 +1146,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: '', description: 'Update tenant settings', + source_type: 'sapi', }, integrations: { Auth0: false, @@ -1161,7 +1161,6 @@ export const data = [ batch: [ { type: 'track', - source_type: 'gd_tenant_update', event: 'Guardian tenant update', sentAt: '2022-10-31T06:15:25.196Z', userId: 'google-oauth2|123456', @@ -1221,6 +1220,7 @@ export const data = [ }, }, description: 'Guardian - Updates tenant settings', + source_type: 'gd_tenant_update', }, integrations: { Auth0: false, @@ -1290,7 +1290,6 @@ export const data = [ batch: [ { type: 'identify', - source_type: 'ss', sentAt: '2022-10-31T05:57:06.859Z', traits: { connection: 'Username-Password-Authentication', @@ -1327,6 +1326,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: 'All Applications', description: '', + source_type: 'ss', }, integrations: { Auth0: false, @@ -1410,7 +1410,6 @@ export const data = [ batch: [ { type: 'identify', - source_type: 'ss', userId: '', anonymousId: '97fcd7b2-cc24-47d7-b776-057b7b199513', sentAt: '2022-10-31T05:57:06.859Z', @@ -1447,6 +1446,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: 'All Applications', description: '', + source_type: 'ss', }, integrations: { Auth0: false, @@ -1461,7 +1461,6 @@ export const data = [ batch: [ { type: 'track', - source_type: 'sapi', event: 'Success API Operation', sentAt: '2022-10-31T05:57:06.874Z', anonymousId: '97fcd7b2-cc24-47d7-b776-057b7b199513', @@ -1482,6 +1481,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: '', description: 'Create a User', + source_type: 'sapi', }, integrations: { Auth0: false, From 04d078310d4b102588f35cf9eb2bc34bf18b23ca Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Fri, 14 Jun 2024 16:11:36 +0530 Subject: [PATCH 4/4] feat: cm360 enhanced conversions (#3414) * feat: cm360 enhanced conversions * refactor: added null, empty checks --- package-lock.json | 6 + package.json | 1 + .../destinations/campaign_manager/config.js | 4 + ...mpaignManagerEnhancedConversionConfig.json | 59 ++ .../campaign_manager/transform.js | 15 +- src/v0/destinations/campaign_manager/util.js | 106 ++++ .../campaign_manager/util.test.js | 36 +- src/v0/util/data/GenericFieldMapping.json | 8 +- .../campaign_manager/processor/data.ts | 513 ++++++++++++++++++ 9 files changed, 744 insertions(+), 4 deletions(-) create mode 100644 src/v0/destinations/campaign_manager/data/CampaignManagerEnhancedConversionConfig.json diff --git a/package-lock.json b/package-lock.json index 3166828d24..1e6b2c7341 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,6 +48,7 @@ "koa": "^2.14.1", "koa-bodyparser": "^4.4.0", "koa2-swagger-ui": "^5.7.0", + "libphonenumber-js": "^1.11.1", "lodash": "^4.17.21", "match-json": "^1.3.5", "md5": "^2.3.0", @@ -14839,6 +14840,11 @@ "node": ">= 0.8.0" } }, + "node_modules/libphonenumber-js": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.11.1.tgz", + "integrity": "sha512-Wze1LPwcnzvcKGcRHFGFECTaLzxOtujwpf924difr5zniyYv1C2PiW0419qDR7m8lKDxsImu5mwxFuXhXpjmvw==" + }, "node_modules/lilconfig": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", diff --git a/package.json b/package.json index 855c7ae3a3..304ed36d13 100644 --- a/package.json +++ b/package.json @@ -93,6 +93,7 @@ "koa": "^2.14.1", "koa-bodyparser": "^4.4.0", "koa2-swagger-ui": "^5.7.0", + "libphonenumber-js": "^1.11.1", "lodash": "^4.17.21", "match-json": "^1.3.5", "md5": "^2.3.0", diff --git a/src/v0/destinations/campaign_manager/config.js b/src/v0/destinations/campaign_manager/config.js index b3a9531347..5ea1972a84 100644 --- a/src/v0/destinations/campaign_manager/config.js +++ b/src/v0/destinations/campaign_manager/config.js @@ -7,6 +7,10 @@ const ConfigCategories = { type: 'track', name: 'CampaignManagerTrackConfig', }, + ENHANCED_CONVERSION: { + type: 'track', + name: 'CampaignManagerEnhancedConversionConfig', + }, }; const MAX_BATCH_CONVERSATIONS_SIZE = 1000; diff --git a/src/v0/destinations/campaign_manager/data/CampaignManagerEnhancedConversionConfig.json b/src/v0/destinations/campaign_manager/data/CampaignManagerEnhancedConversionConfig.json new file mode 100644 index 0000000000..0bce0019dd --- /dev/null +++ b/src/v0/destinations/campaign_manager/data/CampaignManagerEnhancedConversionConfig.json @@ -0,0 +1,59 @@ +[ + { + "destKey": "hashedEmail", + "sourceKeys": "emailOnly", + "sourceFromGenericMap": true + }, + { + "destKey": "hashedPhoneNumber", + "sourceKeys": "phone", + "sourceFromGenericMap": true + }, + { + "destKey": "addressInfo.hashedFirstName", + "sourceKeys": "firstName", + "sourceFromGenericMap": true + }, + { + "destKey": "addressInfo.hashedLastName", + "sourceKeys": "lastName", + "sourceFromGenericMap": true + }, + { + "destKey": "addressInfo.hashedStreetAddress", + "sourceKeys": "street", + "sourceFromGenericMap": true + }, + { + "destKey": "addressInfo.city", + "sourceKeys": [ + "traits.city", + "traits.address.city", + "context.traits.city", + "context.traits.address.city" + ] + }, + { + "destKey": "addressInfo.state", + "sourceKeys": [ + "traits.state", + "traits.address.state", + "context.traits.state", + "context.traits.address.state" + ] + }, + { + "destKey": "addressInfo.countryCode", + "sourceKeys": [ + "traits.country", + "traits.address.country", + "context.traits.country", + "context.traits.address.country" + ] + }, + { + "destKey": "addressInfo.postalCode", + "sourceKeys": "zipcode", + "sourceFromGenericMap": true + } +] diff --git a/src/v0/destinations/campaign_manager/transform.js b/src/v0/destinations/campaign_manager/transform.js index 403a79a971..ef208df5d1 100644 --- a/src/v0/destinations/campaign_manager/transform.js +++ b/src/v0/destinations/campaign_manager/transform.js @@ -12,6 +12,7 @@ const { handleRtTfSingleEventError, getAccessToken, } = require('../../util'); +const { CommonUtils } = require('../../../util/common'); const { ConfigCategories, @@ -22,7 +23,7 @@ const { MAX_BATCH_CONVERSATIONS_SIZE, } = require('./config'); -const { convertToMicroseconds } = require('./util'); +const { convertToMicroseconds, prepareUserIdentifiers } = require('./util'); const { JSON_MIME_TYPE } = require('../../util/constant'); function isEmptyObject(obj) { @@ -105,6 +106,18 @@ function processTrack(message, metadata, destination) { } } + if ( + destination.Config.enableEnhancedConversions && + message.properties.requestType === 'batchupdate' + ) { + const userIdentifiers = CommonUtils.toArray( + prepareUserIdentifiers(message, destination.Config.isHashingRequired ?? true), + ); + if (userIdentifiers.length > 0) { + requestJson.userIdentifiers = userIdentifiers; + } + } + const endpointUrl = prepareUrl(message, destination); return buildResponse( requestJson, diff --git a/src/v0/destinations/campaign_manager/util.js b/src/v0/destinations/campaign_manager/util.js index 434322440f..fee4c8188d 100644 --- a/src/v0/destinations/campaign_manager/util.js +++ b/src/v0/destinations/campaign_manager/util.js @@ -1,3 +1,14 @@ +const { parsePhoneNumber } = require('libphonenumber-js'); +const sha256 = require('sha256'); +const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { + constructPayload, + isDefinedAndNotNull, + removeUndefinedAndNullValues, + isEmptyObject, +} = require('../../util'); +const { ConfigCategories, mappingConfig } = require('./config'); + function convertToMicroseconds(input) { const timestamp = Date.parse(input); @@ -28,6 +39,101 @@ function convertToMicroseconds(input) { return timestamp; } +const normalizeEmail = (email) => { + const domains = ['@gmail.com', '@googlemail.com']; + + const matchingDomain = domains.find((domain) => email.endsWith(domain)); + + if (matchingDomain) { + const localPart = email.split('@')[0].replace(/\./g, ''); + return `${localPart}${matchingDomain}`; + } + + return email; +}; + +const normalizePhone = (phone, countryCode) => { + const phoneNumberObject = parsePhoneNumber(phone, countryCode); + if (phoneNumberObject && phoneNumberObject.isValid()) { + return phoneNumberObject.format('E.164'); + } + throw new InstrumentationError('Invalid phone number'); +}; + +// ref:- https://developers.google.com/doubleclick-advertisers/guides/conversions_ec#hashing +const normalizeAndHash = (key, value, options) => { + if (!isDefinedAndNotNull(value)) return value; + + let normalizedValue; + const trimmedValue = value.trim().toLowerCase(); + switch (key) { + case 'hashedEmail': + normalizedValue = normalizeEmail(trimmedValue); + break; + case 'hashedPhoneNumber': + normalizedValue = normalizePhone(trimmedValue, options.countryCode); + break; + case 'hashedFirstName': + case 'hashedLastName': + case 'hashedStreetAddress': + normalizedValue = trimmedValue; + break; + default: + return value; + } + return sha256(normalizedValue); +}; + +const prepareUserIdentifiers = (message, isHashingRequired) => { + const payload = constructPayload( + message, + mappingConfig[ConfigCategories.ENHANCED_CONVERSION.name], + ); + + if (isHashingRequired) { + payload.hashedEmail = normalizeAndHash('hashedEmail', payload.hashedEmail); + payload.hashedPhoneNumber = normalizeAndHash('hashedPhoneNumber', payload.hashedPhoneNumber, { + options: payload.addressInfo?.countryCode, + }); + + if (!isEmptyObject(payload.addressInfo)) { + payload.addressInfo.hashedFirstName = normalizeAndHash( + 'hashedFirstName', + payload.addressInfo.hashedFirstName, + ); + + payload.addressInfo.hashedLastName = normalizeAndHash( + 'hashedLastName', + payload.addressInfo.hashedLastName, + ); + + payload.addressInfo.hashedStreetAddress = normalizeAndHash( + 'hashedStreetAddress', + payload.addressInfo.hashedStreetAddress, + ); + } + } + + const userIdentifiers = []; + if (isDefinedAndNotNull(payload.hashedEmail)) { + userIdentifiers.push({ hashedEmail: payload.hashedEmail }); + } + if (isDefinedAndNotNull(payload.hashedPhoneNumber)) { + userIdentifiers.push({ hashedPhoneNumber: payload.hashedPhoneNumber }); + } + + payload.addressInfo = removeUndefinedAndNullValues(payload.addressInfo); + if (!isEmptyObject(payload.addressInfo)) { + userIdentifiers.push({ addressInfo: payload.addressInfo }); + } + + return userIdentifiers; +}; + module.exports = { convertToMicroseconds, + normalizeEmail, + normalizePhone, + normalizeAndHash, + prepareUserIdentifiers, }; diff --git a/src/v0/destinations/campaign_manager/util.test.js b/src/v0/destinations/campaign_manager/util.test.js index 8f69b57a9f..bda63f5806 100644 --- a/src/v0/destinations/campaign_manager/util.test.js +++ b/src/v0/destinations/campaign_manager/util.test.js @@ -1,4 +1,10 @@ -const { convertToMicroseconds } = require('./util'); +const sha256 = require('sha256'); +const { + convertToMicroseconds, + normalizeEmail, + normalizePhone, + normalizeAndHash, +} = require('./util'); describe('convertToMicroseconds utility test', () => { it('ISO 8601 input', () => { @@ -21,3 +27,31 @@ describe('convertToMicroseconds utility test', () => { expect(convertToMicroseconds('1697013935000')).toEqual(1697013935000000); }); }); + +describe('normalizeEmail', () => { + it('should remove dots from the local part for gmail.com addresses', () => { + const email = 'example.user@gmail.com'; + const normalized = normalizeEmail(email); + expect(normalized).toBe('exampleuser@gmail.com'); + }); + + it('should return the same email if no google domain is present', () => { + const email = 'exampleuser@exampl.com'; + const normalized = normalizeEmail(email); + expect(normalized).toBe('exampleuser@exampl.com'); + }); +}); + +describe('normalizePhone', () => { + it('should return a valid E.164 formatted phone number when provided with correct inputs', () => { + const validPhone = '4155552671'; + const countryCode = 'US'; + expect(normalizePhone(validPhone, countryCode)).toBe('+14155552671'); + }); + + it('should throw an InstrumentationError when the phone number is too short or too long', () => { + const invalidPhone = '123'; + const countryCode = 'US'; + expect(() => normalizePhone(invalidPhone, countryCode)).toThrow('Invalid phone number'); + }); +}); diff --git a/src/v0/util/data/GenericFieldMapping.json b/src/v0/util/data/GenericFieldMapping.json index 0a7b309d89..b903e6587d 100644 --- a/src/v0/util/data/GenericFieldMapping.json +++ b/src/v0/util/data/GenericFieldMapping.json @@ -72,12 +72,16 @@ "traits.DOB", "context.traits.DOB" ], - "state": ["traits.state", "context.traits.state"], "country": ["traits.country", "context.traits.country"], "region": ["traits.region", "context.traits.region"], "city": ["traits.address.city", "context.traits.address.city"], - + "street": [ + "traits.street", + "traits.address.street", + "context.traits.street", + "context.traits.address.street" + ], "avatar": [ "traits.avatar", "context.traits.avatar", diff --git a/test/integrations/destinations/campaign_manager/processor/data.ts b/test/integrations/destinations/campaign_manager/processor/data.ts index beff44c928..9aa41691c6 100644 --- a/test/integrations/destinations/campaign_manager/processor/data.ts +++ b/test/integrations/destinations/campaign_manager/processor/data.ts @@ -827,4 +827,517 @@ export const data = [ }, }, }, + { + name: 'campaign_manager', + description: 'Test 6: Enhanced Conversions with un-hashed data in request payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + id: '0572f78fa49c648e', + name: 'generic_x86_arm', + type: 'Android', + model: 'AOSP on IA Emulator', + manufacturer: 'Google', + adTrackingEnabled: true, + advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8', + }, + traits: { + email: 'alex@example.com', + phone: '+1-202-555-0146', + firstName: 'John', + lastName: 'Gomes', + city: 'London', + state: 'England', + country: 'GB', + postalCode: 'EC3M', + street: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + }, + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: '34245', + floodlightConfigurationId: '213123123', + ordinal: 'string', + floodlightActivityId: '456543345245', + value: '756', + encryptedUserIdCandidates: ['dfghjbnm'], + quantity: '455678', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + encryptionEntityType: 'DCM_ACCOUNT', + requestType: 'batchupdate', + }, + type: 'track', + event: 'event test', + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + profileId: '5343234', + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + enableEnhancedConversions: true, + isHashingRequired: true, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate', + headers: { + Authorization: 'Bearer dummyApiToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + kind: 'dfareporting#conversionsBatchUpdateRequest', + encryptionInfo: { + encryptionEntityType: 'DCM_ACCOUNT', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + kind: 'dfareporting#encryptionInfo', + }, + conversions: [ + { + floodlightConfigurationId: '213123123', + ordinal: 'string', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 756, + encryptedUserIdCandidates: ['dfghjbnm'], + nonPersonalizedAd: false, + treatmentForUnderage: false, + userIdentifiers: [ + { + hashedEmail: + '6db61e6dcbcf2390e4a46af426f26a133a3bee45021422fc7ae86e9136f14110', + }, + { + hashedPhoneNumber: + 'ec7e6b85f24fa6b796f1017236463f1b7160fbdc5e663e39ab363b6d6fe30b9f', + }, + { + addressInfo: { + hashedFirstName: + '96d9632f363564cc3032521409cf22a852f2032eec099ed5967c0d000cec607a', + hashedLastName: + '12918b23d69d698324a78a8ab8f5060fdb25537ea9620f956d39adca151c3ef9', + hashedStreetAddress: + '5c100d86e9f40bb62a85ca821ff93d96aff6b0dc4c792794c4a4d51ec9246eff', + city: 'London', + state: 'England', + postalCode: 'EC3M', + countryCode: 'GB', + }, + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'campaign_manager', + description: 'Test 7: Enhanced Conversions with hashed data in request payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + id: '0572f78fa49c648e', + name: 'generic_x86_arm', + type: 'Android', + model: 'AOSP on IA Emulator', + manufacturer: 'Google', + adTrackingEnabled: true, + advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8', + }, + traits: { + email: '6db61e6dcbcf2390e4a46af426f26a133a3bee45021422fc7ae86e9136f14110', + phone: '', + firstName: '96d9632f363564cc3032521409cf22a852f2032eec099ed5967c0d000cec607a', + lastName: '12918b23d69d698324a78a8ab8f5060fdb25537ea9620f956d39adca151c3ef9', + city: 'London', + state: 'England', + country: 'GB', + postalCode: 'EC3M', + street: '5c100d86e9f40bb62a85ca821ff93d96aff6b0dc4c792794c4a4d51ec9246eff', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + }, + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: '34245', + floodlightConfigurationId: '213123123', + ordinal: 'string', + floodlightActivityId: '456543345245', + value: '756', + encryptedUserIdCandidates: ['dfghjbnm'], + quantity: '455678', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + encryptionEntityType: 'DCM_ACCOUNT', + requestType: 'batchupdate', + }, + type: 'track', + event: 'event test', + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + profileId: '5343234', + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + enableEnhancedConversions: true, + isHashingRequired: false, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate', + headers: { + Authorization: 'Bearer dummyApiToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + kind: 'dfareporting#conversionsBatchUpdateRequest', + encryptionInfo: { + encryptionEntityType: 'DCM_ACCOUNT', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + kind: 'dfareporting#encryptionInfo', + }, + conversions: [ + { + floodlightConfigurationId: '213123123', + ordinal: 'string', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 756, + encryptedUserIdCandidates: ['dfghjbnm'], + nonPersonalizedAd: false, + treatmentForUnderage: false, + userIdentifiers: [ + { + hashedEmail: + '6db61e6dcbcf2390e4a46af426f26a133a3bee45021422fc7ae86e9136f14110', + }, + { + addressInfo: { + hashedFirstName: + '96d9632f363564cc3032521409cf22a852f2032eec099ed5967c0d000cec607a', + hashedLastName: + '12918b23d69d698324a78a8ab8f5060fdb25537ea9620f956d39adca151c3ef9', + hashedStreetAddress: + '5c100d86e9f40bb62a85ca821ff93d96aff6b0dc4c792794c4a4d51ec9246eff', + city: 'London', + state: 'England', + postalCode: 'EC3M', + countryCode: 'GB', + }, + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'campaign_manager', + description: 'Test 8: Enhanced Conversions with no traits in request payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + id: '0572f78fa49c648e', + name: 'generic_x86_arm', + type: 'Android', + model: 'AOSP on IA Emulator', + manufacturer: 'Google', + adTrackingEnabled: true, + advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + }, + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: '34245', + floodlightConfigurationId: '213123123', + ordinal: 'string', + floodlightActivityId: '456543345245', + value: '756', + encryptedUserIdCandidates: ['dfghjbnm'], + quantity: '455678', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + encryptionEntityType: 'DCM_ACCOUNT', + requestType: 'batchupdate', + }, + type: 'track', + event: 'event test', + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + profileId: '5343234', + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + enableEnhancedConversions: true, + isHashingRequired: true, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate', + headers: { + Authorization: 'Bearer dummyApiToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + kind: 'dfareporting#conversionsBatchUpdateRequest', + encryptionInfo: { + encryptionEntityType: 'DCM_ACCOUNT', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + kind: 'dfareporting#encryptionInfo', + }, + conversions: [ + { + floodlightConfigurationId: '213123123', + ordinal: 'string', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 756, + encryptedUserIdCandidates: ['dfghjbnm'], + nonPersonalizedAd: false, + treatmentForUnderage: false, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + statusCode: 200, + }, + ], + }, + }, + }, ];