diff --git a/.github/workflows/verify-server-start.yml b/.github/workflows/verify-server-start.yml new file mode 100644 index 0000000000..6ac5b7be05 --- /dev/null +++ b/.github/workflows/verify-server-start.yml @@ -0,0 +1,34 @@ +name: Verify Server start + +on: + pull_request: + types: ['opened', 'reopened', 'synchronize'] + +jobs: + check-health: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 1 + + - name: Setup Node + uses: actions/setup-node@v4.0.2 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - name: Install Dependencies + run: npm ci + + - name: Start server + run: npm run build:start & + + - name: Wait for server to start + run: sleep 10 # Adjust the time as necessary for your server to start + + - name: Check server health + run: | + curl --fail http://localhost:9090/health || exit 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b2fff8bce..42b22827da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.69.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.68.2...v1.69.0) (2024-06-10) + + +### Features + +* add request_ip as fallback for mixpanel group call ([#3421](https://github.com/rudderlabs/rudder-transformer/issues/3421)) ([a73ab75](https://github.com/rudderlabs/rudder-transformer/commit/a73ab75032d753b35cb0e18234dcd7289dd1e644)) +* add v3 api support to appsflyer ([#3412](https://github.com/rudderlabs/rudder-transformer/issues/3412)) ([e124470](https://github.com/rudderlabs/rudder-transformer/commit/e124470e82b6aa9934094146d4050af02bb62fff)), closes [#3395](https://github.com/rudderlabs/rudder-transformer/issues/3395) [#3402](https://github.com/rudderlabs/rudder-transformer/issues/3402) +* changes for supporting record event in FB audience ([#3351](https://github.com/rudderlabs/rudder-transformer/issues/3351)) ([ac4a32a](https://github.com/rudderlabs/rudder-transformer/commit/ac4a32ab5e0c7e02a149e81d455666ed24fa01a3)) + + +### Bug Fixes + +* allowing traffic type dynamically for split.io ([#3425](https://github.com/rudderlabs/rudder-transformer/issues/3425)) ([3bea186](https://github.com/rudderlabs/rudder-transformer/commit/3bea186e725ec473ad757760355d6cc9670c4f8c)) +* bugsnag issue fix for zendesk ([#3439](https://github.com/rudderlabs/rudder-transformer/issues/3439)) ([775e8ee](https://github.com/rudderlabs/rudder-transformer/commit/775e8ee55a62ecddb58ff505302e4aabb8bffe24)) + ### [1.68.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.68.1...v1.68.2) (2024-06-06) diff --git a/package-lock.json b/package-lock.json index 5effe0c249..1e6b2c7341 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.68.2", + "version": "1.69.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.68.2", + "version": "1.69.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", @@ -20,7 +20,8 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.8", - "@rudderstack/workflow-engine": "^0.7.9", + "@rudderstack/json-template-engine": "^0.13.2", + "@rudderstack/workflow-engine": "^0.8.2", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", @@ -47,6 +48,7 @@ "koa": "^2.14.1", "koa-bodyparser": "^4.4.0", "koa2-swagger-ui": "^5.7.0", + "libphonenumber-js": "^1.11.1", "lodash": "^4.17.21", "match-json": "^1.3.5", "md5": "^2.3.0", @@ -60,6 +62,7 @@ "parse-static-imports": "^1.1.0", "prom-client": "^14.2.0", "qs": "^6.11.1", + "rs-jsonpath": "^1.1.2", "rudder-transformer-cdk": "^1.4.11", "set-value": "^4.1.0", "sha256": "^0.2.0", @@ -78,6 +81,7 @@ "@digitalroute/cz-conventional-changelog-for-jira": "^8.0.1", "@types/fast-json-stable-stringify": "^2.1.0", "@types/jest": "^29.5.1", + "@types/jsonpath": "^0.2.4", "@types/koa": "^2.13.6", "@types/koa-bodyparser": "^4.3.10", "@types/lodash": "^4.14.197", @@ -118,6 +122,27 @@ "typescript": "^5.0.4" } }, + "../jsonpath": { + "name": "rs-jsonpath", + "version": "1.1.2", + "extraneous": true, + "license": "MIT", + "dependencies": { + "esprima": "1.2.2", + "static-eval": "2.0.2", + "underscore": "1.12.1" + }, + "devDependencies": { + "grunt": "0.4.5", + "grunt-browserify": "3.8.0", + "grunt-cli": "0.1.13", + "grunt-contrib-uglify": "0.9.1", + "jison": "0.4.13", + "jscs": "1.10.0", + "jshint": "2.6.0", + "mocha": "2.1.0" + } + }, "node_modules/@aashutoshrathi/word-wrap": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", @@ -4458,17 +4483,17 @@ } }, "node_modules/@rudderstack/json-template-engine": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.10.5.tgz", - "integrity": "sha512-PasCK5RDwiRHsFhAb3w0n+8JPRYcZTffe2l+M/wtzvqU+12NPj3YTEIaMWkhogY6AmPYswAaMX/kr+4j7dKiUA==" + "version": "0.13.2", + "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.13.2.tgz", + "integrity": "sha512-uEyMv/qjm/mP5V8EifJzolvFLtka/dacmvwo9Xk3+MnEbsNN0YLu7Z/qWeyXeDF5chvy8JfaqV8lNgO3SxVG7g==" }, "node_modules/@rudderstack/workflow-engine": { - "version": "0.7.9", - "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.7.9.tgz", - "integrity": "sha512-uMELZk7UXs40bgQkIk7fIVrfHo/5ld+5I5kYgZt5rcT65H9aNpWjnNRnsKH9dgu+oxiBFAMassZq5ko4hpEdIQ==", + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.8.2.tgz", + "integrity": "sha512-cjn3J8CUarAE3cbASRvkf7A2745Clzkw/ffqGLzD8+9KvTN6mC28Pm9c5169LPDmt+NMUMw0W5xHgNO3cV9eqQ==", "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", - "@rudderstack/json-template-engine": "^0.10.5", + "@rudderstack/json-template-engine": "^0.13.2", "jsonata": "^2.0.5", "lodash": "^4.17.21", "object-sizeof": "^2.6.4", @@ -5407,6 +5432,12 @@ "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==" }, + "node_modules/@types/jsonpath": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@types/jsonpath/-/jsonpath-0.2.4.tgz", + "integrity": "sha512-K3hxB8Blw0qgW6ExKgMbXQv2UPZBoE2GqLpVY+yr7nMD2Pq86lsuIzyAaiQ7eMqFL5B6di6pxSkogLJEyEHoGA==", + "dev": true + }, "node_modules/@types/keygrip": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.6.tgz", @@ -6529,11 +6560,11 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -10016,7 +10047,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, "bin": { "esparse": "bin/esparse.js", "esvalidate": "bin/esvalidate.js" @@ -10362,9 +10392,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -14810,6 +14840,11 @@ "node": ">= 0.8.0" } }, + "node_modules/libphonenumber-js": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.11.1.tgz", + "integrity": "sha512-Wze1LPwcnzvcKGcRHFGFECTaLzxOtujwpf924difr5zniyYv1C2PiW0419qDR7m8lKDxsImu5mwxFuXhXpjmvw==" + }, "node_modules/lilconfig": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", @@ -18603,6 +18638,28 @@ "node": ">=18.0" } }, + "node_modules/rs-jsonpath": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/rs-jsonpath/-/rs-jsonpath-1.1.2.tgz", + "integrity": "sha512-IQzlqtVyZniK7aOtpKGrv7BvkamSvLJkIhRGoKKDQLppNJe94BVHqpxNRjw/2042nGjtC3vyfCWyHe+3DlWgWA==", + "dependencies": { + "esprima": "1.2.2", + "static-eval": "2.0.2", + "underscore": "1.12.1" + } + }, + "node_modules/rs-jsonpath/node_modules/esprima": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.2.2.tgz", + "integrity": "sha512-+JpPZam9w5DuJ3Q67SqsMGtiHKENSMRVoxvArfJZK01/BfLEObtZ6orJa/MtoGNR/rfMgp5837T41PAmTwAv/A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/rudder-transformer-cdk": { "version": "1.4.11", "resolved": "https://registry.npmjs.org/rudder-transformer-cdk/-/rudder-transformer-cdk-1.4.11.tgz", @@ -19480,6 +19537,91 @@ "node": ">=10" } }, + "node_modules/static-eval": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/static-eval/-/static-eval-2.0.2.tgz", + "integrity": "sha512-N/D219Hcr2bPjLxPiV+TQE++Tsmrady7TqAJugLy7Xk1EumfDWS/f5dtBbkRCGE7wKKXuYockQoj8Rm2/pVKyg==", + "dependencies": { + "escodegen": "^1.8.1" + } + }, + "node_modules/static-eval/node_modules/escodegen": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", + "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=4.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/static-eval/node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/static-eval/node_modules/optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/static-eval/node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/static-eval/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-eval/node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "dependencies": { + "prelude-ls": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/stats-accumulator": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/stats-accumulator/-/stats-accumulator-1.1.3.tgz", @@ -20420,6 +20562,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/underscore": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz", + "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==" + }, "node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", @@ -20810,7 +20957,6 @@ "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", - "dev": true, "engines": { "node": ">=0.10.0" } diff --git a/package.json b/package.json index 3aa3c017ff..304ed36d13 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.68.2", + "version": "1.69.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { @@ -65,7 +65,8 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.8", - "@rudderstack/workflow-engine": "^0.7.9", + "@rudderstack/json-template-engine": "^0.13.2", + "@rudderstack/workflow-engine": "^0.8.2", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", @@ -92,6 +93,7 @@ "koa": "^2.14.1", "koa-bodyparser": "^4.4.0", "koa2-swagger-ui": "^5.7.0", + "libphonenumber-js": "^1.11.1", "lodash": "^4.17.21", "match-json": "^1.3.5", "md5": "^2.3.0", @@ -105,6 +107,7 @@ "parse-static-imports": "^1.1.0", "prom-client": "^14.2.0", "qs": "^6.11.1", + "rs-jsonpath": "^1.1.2", "rudder-transformer-cdk": "^1.4.11", "set-value": "^4.1.0", "sha256": "^0.2.0", @@ -123,6 +126,7 @@ "@digitalroute/cz-conventional-changelog-for-jira": "^8.0.1", "@types/fast-json-stable-stringify": "^2.1.0", "@types/jest": "^29.5.1", + "@types/jsonpath": "^0.2.4", "@types/koa": "^2.13.6", "@types/koa-bodyparser": "^4.3.10", "@types/lodash": "^4.14.197", diff --git a/src/v0/destinations/campaign_manager/config.js b/src/v0/destinations/campaign_manager/config.js index b3a9531347..5ea1972a84 100644 --- a/src/v0/destinations/campaign_manager/config.js +++ b/src/v0/destinations/campaign_manager/config.js @@ -7,6 +7,10 @@ const ConfigCategories = { type: 'track', name: 'CampaignManagerTrackConfig', }, + ENHANCED_CONVERSION: { + type: 'track', + name: 'CampaignManagerEnhancedConversionConfig', + }, }; const MAX_BATCH_CONVERSATIONS_SIZE = 1000; diff --git a/src/v0/destinations/campaign_manager/data/CampaignManagerEnhancedConversionConfig.json b/src/v0/destinations/campaign_manager/data/CampaignManagerEnhancedConversionConfig.json new file mode 100644 index 0000000000..0bce0019dd --- /dev/null +++ b/src/v0/destinations/campaign_manager/data/CampaignManagerEnhancedConversionConfig.json @@ -0,0 +1,59 @@ +[ + { + "destKey": "hashedEmail", + "sourceKeys": "emailOnly", + "sourceFromGenericMap": true + }, + { + "destKey": "hashedPhoneNumber", + "sourceKeys": "phone", + "sourceFromGenericMap": true + }, + { + "destKey": "addressInfo.hashedFirstName", + "sourceKeys": "firstName", + "sourceFromGenericMap": true + }, + { + "destKey": "addressInfo.hashedLastName", + "sourceKeys": "lastName", + "sourceFromGenericMap": true + }, + { + "destKey": "addressInfo.hashedStreetAddress", + "sourceKeys": "street", + "sourceFromGenericMap": true + }, + { + "destKey": "addressInfo.city", + "sourceKeys": [ + "traits.city", + "traits.address.city", + "context.traits.city", + "context.traits.address.city" + ] + }, + { + "destKey": "addressInfo.state", + "sourceKeys": [ + "traits.state", + "traits.address.state", + "context.traits.state", + "context.traits.address.state" + ] + }, + { + "destKey": "addressInfo.countryCode", + "sourceKeys": [ + "traits.country", + "traits.address.country", + "context.traits.country", + "context.traits.address.country" + ] + }, + { + "destKey": "addressInfo.postalCode", + "sourceKeys": "zipcode", + "sourceFromGenericMap": true + } +] diff --git a/src/v0/destinations/campaign_manager/transform.js b/src/v0/destinations/campaign_manager/transform.js index 403a79a971..ef208df5d1 100644 --- a/src/v0/destinations/campaign_manager/transform.js +++ b/src/v0/destinations/campaign_manager/transform.js @@ -12,6 +12,7 @@ const { handleRtTfSingleEventError, getAccessToken, } = require('../../util'); +const { CommonUtils } = require('../../../util/common'); const { ConfigCategories, @@ -22,7 +23,7 @@ const { MAX_BATCH_CONVERSATIONS_SIZE, } = require('./config'); -const { convertToMicroseconds } = require('./util'); +const { convertToMicroseconds, prepareUserIdentifiers } = require('./util'); const { JSON_MIME_TYPE } = require('../../util/constant'); function isEmptyObject(obj) { @@ -105,6 +106,18 @@ function processTrack(message, metadata, destination) { } } + if ( + destination.Config.enableEnhancedConversions && + message.properties.requestType === 'batchupdate' + ) { + const userIdentifiers = CommonUtils.toArray( + prepareUserIdentifiers(message, destination.Config.isHashingRequired ?? true), + ); + if (userIdentifiers.length > 0) { + requestJson.userIdentifiers = userIdentifiers; + } + } + const endpointUrl = prepareUrl(message, destination); return buildResponse( requestJson, diff --git a/src/v0/destinations/campaign_manager/util.js b/src/v0/destinations/campaign_manager/util.js index 434322440f..fee4c8188d 100644 --- a/src/v0/destinations/campaign_manager/util.js +++ b/src/v0/destinations/campaign_manager/util.js @@ -1,3 +1,14 @@ +const { parsePhoneNumber } = require('libphonenumber-js'); +const sha256 = require('sha256'); +const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { + constructPayload, + isDefinedAndNotNull, + removeUndefinedAndNullValues, + isEmptyObject, +} = require('../../util'); +const { ConfigCategories, mappingConfig } = require('./config'); + function convertToMicroseconds(input) { const timestamp = Date.parse(input); @@ -28,6 +39,101 @@ function convertToMicroseconds(input) { return timestamp; } +const normalizeEmail = (email) => { + const domains = ['@gmail.com', '@googlemail.com']; + + const matchingDomain = domains.find((domain) => email.endsWith(domain)); + + if (matchingDomain) { + const localPart = email.split('@')[0].replace(/\./g, ''); + return `${localPart}${matchingDomain}`; + } + + return email; +}; + +const normalizePhone = (phone, countryCode) => { + const phoneNumberObject = parsePhoneNumber(phone, countryCode); + if (phoneNumberObject && phoneNumberObject.isValid()) { + return phoneNumberObject.format('E.164'); + } + throw new InstrumentationError('Invalid phone number'); +}; + +// ref:- https://developers.google.com/doubleclick-advertisers/guides/conversions_ec#hashing +const normalizeAndHash = (key, value, options) => { + if (!isDefinedAndNotNull(value)) return value; + + let normalizedValue; + const trimmedValue = value.trim().toLowerCase(); + switch (key) { + case 'hashedEmail': + normalizedValue = normalizeEmail(trimmedValue); + break; + case 'hashedPhoneNumber': + normalizedValue = normalizePhone(trimmedValue, options.countryCode); + break; + case 'hashedFirstName': + case 'hashedLastName': + case 'hashedStreetAddress': + normalizedValue = trimmedValue; + break; + default: + return value; + } + return sha256(normalizedValue); +}; + +const prepareUserIdentifiers = (message, isHashingRequired) => { + const payload = constructPayload( + message, + mappingConfig[ConfigCategories.ENHANCED_CONVERSION.name], + ); + + if (isHashingRequired) { + payload.hashedEmail = normalizeAndHash('hashedEmail', payload.hashedEmail); + payload.hashedPhoneNumber = normalizeAndHash('hashedPhoneNumber', payload.hashedPhoneNumber, { + options: payload.addressInfo?.countryCode, + }); + + if (!isEmptyObject(payload.addressInfo)) { + payload.addressInfo.hashedFirstName = normalizeAndHash( + 'hashedFirstName', + payload.addressInfo.hashedFirstName, + ); + + payload.addressInfo.hashedLastName = normalizeAndHash( + 'hashedLastName', + payload.addressInfo.hashedLastName, + ); + + payload.addressInfo.hashedStreetAddress = normalizeAndHash( + 'hashedStreetAddress', + payload.addressInfo.hashedStreetAddress, + ); + } + } + + const userIdentifiers = []; + if (isDefinedAndNotNull(payload.hashedEmail)) { + userIdentifiers.push({ hashedEmail: payload.hashedEmail }); + } + if (isDefinedAndNotNull(payload.hashedPhoneNumber)) { + userIdentifiers.push({ hashedPhoneNumber: payload.hashedPhoneNumber }); + } + + payload.addressInfo = removeUndefinedAndNullValues(payload.addressInfo); + if (!isEmptyObject(payload.addressInfo)) { + userIdentifiers.push({ addressInfo: payload.addressInfo }); + } + + return userIdentifiers; +}; + module.exports = { convertToMicroseconds, + normalizeEmail, + normalizePhone, + normalizeAndHash, + prepareUserIdentifiers, }; diff --git a/src/v0/destinations/campaign_manager/util.test.js b/src/v0/destinations/campaign_manager/util.test.js index 8f69b57a9f..bda63f5806 100644 --- a/src/v0/destinations/campaign_manager/util.test.js +++ b/src/v0/destinations/campaign_manager/util.test.js @@ -1,4 +1,10 @@ -const { convertToMicroseconds } = require('./util'); +const sha256 = require('sha256'); +const { + convertToMicroseconds, + normalizeEmail, + normalizePhone, + normalizeAndHash, +} = require('./util'); describe('convertToMicroseconds utility test', () => { it('ISO 8601 input', () => { @@ -21,3 +27,31 @@ describe('convertToMicroseconds utility test', () => { expect(convertToMicroseconds('1697013935000')).toEqual(1697013935000000); }); }); + +describe('normalizeEmail', () => { + it('should remove dots from the local part for gmail.com addresses', () => { + const email = 'example.user@gmail.com'; + const normalized = normalizeEmail(email); + expect(normalized).toBe('exampleuser@gmail.com'); + }); + + it('should return the same email if no google domain is present', () => { + const email = 'exampleuser@exampl.com'; + const normalized = normalizeEmail(email); + expect(normalized).toBe('exampleuser@exampl.com'); + }); +}); + +describe('normalizePhone', () => { + it('should return a valid E.164 formatted phone number when provided with correct inputs', () => { + const validPhone = '4155552671'; + const countryCode = 'US'; + expect(normalizePhone(validPhone, countryCode)).toBe('+14155552671'); + }); + + it('should throw an InstrumentationError when the phone number is too short or too long', () => { + const invalidPhone = '123'; + const countryCode = 'US'; + expect(() => normalizePhone(invalidPhone, countryCode)).toThrow('Invalid phone number'); + }); +}); diff --git a/src/v0/destinations/ga4/transform.js b/src/v0/destinations/ga4/transform.js index 5280a46dab..e4dad80564 100644 --- a/src/v0/destinations/ga4/transform.js +++ b/src/v0/destinations/ga4/transform.js @@ -1,25 +1,15 @@ const get = require('get-value'); -const { - ConfigurationError, - InstrumentationError, - UnsupportedEventError, -} = require('@rudderstack/integrations-lib'); +const { InstrumentationError, UnsupportedEventError } = require('@rudderstack/integrations-lib'); const { EventType } = require('../../../constants'); const { isEmptyObject, constructPayload, getIntegrationsObj, isHybridModeEnabled, - isDefinedAndNotNull, - defaultRequestConfig, - defaultPostRequestConfig, - getDestinationExternalID, removeUndefinedAndNullValues, } = require('../../util'); const { - ENDPOINT, mappingConfig, - DEBUG_ENDPOINT, ConfigCategory, trackCommonConfig, VALID_ITEM_OR_PRODUCT_PROPERTIES, @@ -36,33 +26,12 @@ const { GA4_PARAMETERS_EXCLUSION, GA4_RESERVED_PARAMETER_EXCLUSION, removeReservedParameterPrefixNames, + basicValidation, + addClientDetails, + buildDeliverablePayload, + basicConfigvalidaiton, } = require('./utils'); -const { JSON_MIME_TYPE } = require('../../util/constant'); - -/** - * returns client_id - * @param {*} message - * @returns - */ -const getGA4ClientId = (message, Config) => { - let clientId; - - if (isHybridModeEnabled(Config)) { - const integrationsObj = getIntegrationsObj(message, 'ga4'); - if (integrationsObj?.clientId) { - clientId = integrationsObj.clientId; - } - } - - if (!clientId) { - clientId = - getDestinationExternalID(message, 'ga4ClientId') || - get(message, 'anonymousId') || - get(message, 'rudderId'); - } - - return clientId; -}; +require('../../util/constant'); /** * Returns response for GA4 destination @@ -72,14 +41,9 @@ const getGA4ClientId = (message, Config) => { */ const responseBuilder = (message, { Config }) => { let event = get(message, 'event'); - if (!event) { - throw new InstrumentationError('Event name is required'); - } + basicValidation(event); // trim and replace spaces with "_" - if (typeof event !== 'string') { - throw new InstrumentationError('track:: event name should be string'); - } event = event.trim().replace(/\s+/g, '_'); // reserved event names are not allowed @@ -90,25 +54,7 @@ const responseBuilder = (message, { Config }) => { // get common top level rawPayload let rawPayload = constructPayload(message, trackCommonConfig); - switch (Config.typesOfClient) { - case 'gtag': - // gtag.js uses client_id - // GA4 uses it as an identifier to distinguish site visitors. - rawPayload.client_id = getGA4ClientId(message, Config); - if (!isDefinedAndNotNull(rawPayload.client_id)) { - throw new ConfigurationError('ga4ClientId, anonymousId or messageId must be provided'); - } - break; - case 'firebase': - // firebase uses app_instance_id - rawPayload.app_instance_id = getDestinationExternalID(message, 'ga4AppInstanceId'); - if (!isDefinedAndNotNull(rawPayload.app_instance_id)) { - throw new InstrumentationError('ga4AppInstanceId must be provided under externalId'); - } - break; - default: - throw new ConfigurationError('Invalid type of client'); - } + rawPayload = addClientDetails(rawPayload, message, Config); let payload = {}; const eventConfig = ConfigCategory[`${event.toUpperCase()}`]; @@ -248,62 +194,21 @@ const responseBuilder = (message, { Config }) => { payload = removeUndefinedAndNullValues(payload); rawPayload = { ...rawPayload, events: [payload] }; - // build response - const response = defaultRequestConfig(); - response.method = defaultPostRequestConfig.requestMethod; - // if debug_mode is true, we need to send the event to debug validation server - // ref: https://developers.google.com/analytics/devguides/collection/protocol/ga4/validating-events?client_type=firebase#sending_events_for_validation - if (Config.debugMode) { - response.endpoint = DEBUG_ENDPOINT; - } else { - response.endpoint = ENDPOINT; - } - response.headers = { - HOST: 'www.google-analytics.com', - 'Content-Type': JSON_MIME_TYPE, - }; - response.params = { - api_secret: Config.apiSecret, - }; - - // setting response params as per client type - switch (Config.typesOfClient) { - case 'gtag': - response.params.measurement_id = Config.measurementId; - break; - case 'firebase': - response.params.firebase_app_id = Config.firebaseAppId; - break; - default: - break; - } - - response.body.JSON = rawPayload; - return response; + return buildDeliverablePayload(rawPayload, Config); }; const process = (event) => { const { message, destination } = event; const { Config } = destination; - if (!Config.typesOfClient) { - throw new ConfigurationError('Client type not found. Aborting '); - } - if (!Config.apiSecret) { - throw new ConfigurationError('API Secret not found. Aborting '); - } - if (Config.typesOfClient === 'gtag' && !Config.measurementId) { - throw new ConfigurationError('measurementId must be provided. Aborting'); - } - if (Config.typesOfClient === 'firebase' && !Config.firebaseAppId) { - throw new ConfigurationError('firebaseAppId must be provided. Aborting'); - } - if (!message.type) { throw new InstrumentationError('Message Type is not present. Aborting message.'); } + basicConfigvalidaiton(Config); + const messageType = message.type.toLowerCase(); + let response; switch (messageType) { case EventType.TRACK: diff --git a/src/v0/destinations/ga4/utils.js b/src/v0/destinations/ga4/utils.js index ce8afda560..77f78fbfdb 100644 --- a/src/v0/destinations/ga4/utils.js +++ b/src/v0/destinations/ga4/utils.js @@ -1,5 +1,8 @@ +/* eslint-disable no-param-reassign */ +/* eslint-disable no-plusplus */ const get = require('get-value'); -const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); +const { cloneDeep } = require('lodash'); const { isEmpty, constructPayload, @@ -8,9 +11,14 @@ const { extractCustomFields, isDefinedAndNotNull, getIntegrationsObj, + getDestinationExternalID, + isHybridModeEnabled, + defaultPostRequestConfig, + defaultRequestConfig, } = require('../../util'); -const { mappingConfig, ConfigCategory } = require('./config'); +const { mappingConfig, ConfigCategory, DEBUG_ENDPOINT, ENDPOINT } = require('./config'); const { finaliseAnalyticsConsents } = require('../../util/googleUtils'); +const { JSON_MIME_TYPE } = require('../../util/constant'); /** * Reserved event names cannot be used @@ -452,7 +460,136 @@ const prepareUserConsents = (message) => { return consents; }; +const basicValidation = (event) => { + if (!event) { + throw new InstrumentationError('Event name is required'); + } + if (typeof event !== 'string') { + throw new InstrumentationError('track:: event name should be string'); + } +}; + +/** + * returns client_id + * @param {*} message + * @returns + */ +const getGA4ClientId = (message, Config) => { + let clientId; + + if (isHybridModeEnabled(Config)) { + const integrationsObj = getIntegrationsObj(message, 'ga4'); + if (integrationsObj?.clientId) { + clientId = integrationsObj.clientId; + } + } + + if (!clientId) { + clientId = + getDestinationExternalID(message, 'ga4ClientId') || + get(message, 'anonymousId') || + get(message, 'rudderId'); + } + + return clientId; +}; + +const addClientDetails = (payload, message, Config) => { + const { typesOfClient } = Config; + const rawPayload = cloneDeep(payload); + switch (typesOfClient) { + case 'gtag': + // gtag.js uses client_id + // GA4 uses it as an identifier to distinguish site visitors. + rawPayload.client_id = getGA4ClientId(message, Config); + if (!isDefinedAndNotNull(rawPayload.client_id)) { + throw new ConfigurationError('ga4ClientId, anonymousId or messageId must be provided'); + } + break; + case 'firebase': + // firebase uses app_instance_id + rawPayload.app_instance_id = getDestinationExternalID(message, 'ga4AppInstanceId'); + if (!isDefinedAndNotNull(rawPayload.app_instance_id)) { + throw new InstrumentationError('ga4AppInstanceId must be provided under externalId'); + } + break; + default: + throw new ConfigurationError('Invalid type of client'); + } + return rawPayload; +}; + +const buildDeliverablePayload = (payload, Config) => { + // build response + const response = defaultRequestConfig(); + response.method = defaultPostRequestConfig.requestMethod; + // if debug_mode is true, we need to send the event to debug validation server + // ref: https://developers.google.com/analytics/devguides/collection/protocol/ga4/validating-events?client_type=firebase#sending_events_for_validation + if (Config.debugMode) { + response.endpoint = DEBUG_ENDPOINT; + } else { + response.endpoint = ENDPOINT; + } + response.headers = { + HOST: 'www.google-analytics.com', + 'Content-Type': JSON_MIME_TYPE, + }; + response.params = { + api_secret: Config.apiSecret, + }; + + // setting response params as per client type + switch (Config.typesOfClient) { + case 'gtag': + response.params.measurement_id = Config.measurementId; + break; + case 'firebase': + response.params.firebase_app_id = Config.firebaseAppId; + break; + default: + break; + } + + response.body.JSON = payload; + return response; +}; + +const sanitizeUserProperties = (userProperties) => { + Object.keys(userProperties).forEach((key) => { + const propetyValue = userProperties[key]; + if ( + typeof propetyValue === 'string' || + typeof propetyValue === 'number' || + typeof propetyValue === 'boolean' + ) { + delete userProperties[key]; + userProperties[key] = { + value: propetyValue, + }; + } + }); +}; + +const basicConfigvalidaiton = (Config) => { + if (!Config.typesOfClient) { + throw new ConfigurationError('Client type not found. Aborting '); + } + if (!Config.apiSecret) { + throw new ConfigurationError('API Secret not found. Aborting '); + } + if (Config.typesOfClient === 'gtag' && !Config.measurementId) { + throw new ConfigurationError('measurementId must be provided. Aborting'); + } + if (Config.typesOfClient === 'firebase' && !Config.firebaseAppId) { + throw new ConfigurationError('firebaseAppId must be provided. Aborting'); + } +}; + module.exports = { + addClientDetails, + basicValidation, + buildDeliverablePayload, + basicConfigvalidaiton, getItem, getItemList, getItemsArray, @@ -463,6 +600,7 @@ module.exports = { getGA4ExclusionList, prepareUserProperties, getGA4CustomParameters, + sanitizeUserProperties, GA4_PARAMETERS_EXCLUSION, isReservedWebCustomEventName, isReservedWebCustomPrefixName, diff --git a/src/v0/destinations/ga4_v2/customMappingsHandler.js b/src/v0/destinations/ga4_v2/customMappingsHandler.js new file mode 100644 index 0000000000..1eb1c2c868 --- /dev/null +++ b/src/v0/destinations/ga4_v2/customMappingsHandler.js @@ -0,0 +1,165 @@ +const get = require('get-value'); +const { + validateEventName, + basicValidation, + isReservedEventName, + addClientDetails, + removeReservedParameterPrefixNames, + prepareUserConsents, + removeInvalidParams, + GA4_RESERVED_PARAMETER_EXCLUSION, + getGA4CustomParameters, + buildDeliverablePayload, + GA4_PARAMETERS_EXCLUSION, + prepareUserProperties, +} = require('../ga4/utils'); +const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { + removeUndefinedAndNullRecurse, + constructPayload, + isDefinedAndNotNull, + isEmptyObject, + removeUndefinedAndNullValues, + isHybridModeEnabled, + getIntegrationsObj, + applyCustomMappings, +} = require('../../util'); +const { trackCommonConfig, ConfigCategory, mappingConfig } = require('../ga4/config'); + +const findGA4Events = (eventsMapping, event) => { + // Find the event using destructuring and early return + + const validMappings = eventsMapping.filter( + (mapping) => + mapping.rsEventName?.trim().toLowerCase() === event.trim().toLowerCase() && + mapping.destEventName, + ); + // Return an empty object if event not found + return validMappings; +}; + +const handleCustomMappings = (message, Config) => { + const { eventsMapping } = Config; + + let rsEvent = ''; + if (message.type.toString().toLowerCase() === 'track') { + rsEvent = get(message, 'event'); + basicValidation(rsEvent); + } else { + const messageType = get(message, 'type'); + if (typeof messageType !== 'string') { + throw new InstrumentationError(`[GA4]:: Message type ${messageType} is not supported`); + } + // for events other than track we will search with $eventType + // example $track / $page + rsEvent = `$${messageType}`; + } + + const validMappings = findGA4Events(eventsMapping, rsEvent); + + if (validMappings.length === 0) { + // trim and replace spaces with "_" + rsEvent = rsEvent.trim().replace(/\s+/g, '_'); + // reserved event names are not allowed + if (isReservedEventName(rsEvent)) { + throw new InstrumentationError(`[GA4]:: Reserved event name: ${rsEvent} are not allowed`); + } + // validation for ga4 event name + validateEventName(rsEvent); + + // Default mapping + + let rawPayload = constructPayload(message, trackCommonConfig); + + const ga4EventPayload = {}; + + // take optional params parameters for custom events + ga4EventPayload.params = { + ...ga4EventPayload.params, + ...constructPayload(message, mappingConfig[ConfigCategory.TrackPageCommonParamsConfig.name]), + }; + + // all extra parameters passed is incorporated inside params + ga4EventPayload.params = getGA4CustomParameters( + message, + ['properties'], + GA4_RESERVED_PARAMETER_EXCLUSION.concat(GA4_PARAMETERS_EXCLUSION), + ga4EventPayload, + ); + + // Prepare GA4 user properties + const userProperties = prepareUserProperties(message, Config.piiPropertiesToIgnore); + if (!isEmptyObject(userProperties)) { + rawPayload.user_properties = userProperties; + } + + rawPayload = removeUndefinedAndNullValues(rawPayload); + rawPayload = { ...rawPayload, events: [ga4EventPayload] }; + + boilerplateOperations(rawPayload, message, Config, rsEvent); + + return buildDeliverablePayload(rawPayload, Config); + } + + const processedPayloads = validMappings.map((mapping) => { + const eventName = mapping.destEventName; + // reserved event names are not allowed + if (isReservedEventName(eventName)) { + throw new InstrumentationError(`[GA4]:: Reserved event name: ${eventName} are not allowed`); + } + // validation for ga4 event name + validateEventName(eventName); + + // Add common top level payload + let ga4BasicPayload = constructPayload(message, trackCommonConfig); + ga4BasicPayload = addClientDetails(ga4BasicPayload, message, Config); + + const eventPropertiesMappings = mapping.eventProperties || []; + + const ga4MappedPayload = applyCustomMappings(message, eventPropertiesMappings); + + removeUndefinedAndNullRecurse(ga4MappedPayload); + + boilerplateOperations(ga4MappedPayload, message, Config, eventName); + + if (isDefinedAndNotNull(ga4BasicPayload)) { + return { ...ga4BasicPayload, ...ga4MappedPayload }; + } else { + return ga4MappedPayload; + } + }); + + return processedPayloads.map((processedPayload) => + buildDeliverablePayload(processedPayload, Config), + ); +}; + +const boilerplateOperations = (ga4Payload, message, Config, eventName) => { + removeReservedParameterPrefixNames(ga4Payload.events[0].params); + ga4Payload.events[0].name = eventName; + const integrationsObj = getIntegrationsObj(message, 'ga4'); + + if (isHybridModeEnabled(Config) && integrationsObj?.sessionId) { + ga4Payload.events[0].params.session_id = integrationsObj.sessionId; + } + + if (ga4Payload.events[0].params) { + ga4Payload.events[0].params = removeInvalidParams( + removeUndefinedAndNullValues(ga4Payload.events[0].params), + ); + } + + if (isEmptyObject(ga4Payload.events[0].params)) { + delete ga4Payload.events[0].params; + } + + // Prepare GA4 consents + const consents = prepareUserConsents(message); + if (!isEmptyObject(consents)) { + ga4Payload.consent = consents; + } +}; + +module.exports = { + handleCustomMappings, +}; diff --git a/src/v0/destinations/ga4_v2/transform.ts b/src/v0/destinations/ga4_v2/transform.ts new file mode 100644 index 0000000000..76adc00e00 --- /dev/null +++ b/src/v0/destinations/ga4_v2/transform.ts @@ -0,0 +1,25 @@ +import { InstrumentationError, RudderStackEvent } from '@rudderstack/integrations-lib'; +import { ProcessorTransformationRequest } from '../../../types'; +import { handleCustomMappings } from './customMappingsHandler'; +import { process as ga4Process } from '../ga4/transform'; +import { basicConfigvalidaiton } from '../ga4/utils'; + +export function process(event: ProcessorTransformationRequest) { + const { message, destination } = event; + const { Config } = destination; + + const eventPayload = message as RudderStackEvent; + + if (!eventPayload.type) { + throw new InstrumentationError('Message Type is not present. Aborting message.'); + } + + if (eventPayload.type !== 'track') { + return ga4Process(event); + } + + basicConfigvalidaiton(Config); + + // custom mappings flow + return handleCustomMappings(message, Config); +} diff --git a/src/v0/sources/auth0/mapping.json b/src/v0/sources/auth0/mapping.json index bc5869a19b..dcbd389945 100644 --- a/src/v0/sources/auth0/mapping.json +++ b/src/v0/sources/auth0/mapping.json @@ -65,6 +65,6 @@ }, { "sourceKeys": "type", - "destKeys": "source_type" + "destKeys": "properties.source_type" } ] diff --git a/src/v0/util/data/GenericFieldMapping.json b/src/v0/util/data/GenericFieldMapping.json index 0a7b309d89..b903e6587d 100644 --- a/src/v0/util/data/GenericFieldMapping.json +++ b/src/v0/util/data/GenericFieldMapping.json @@ -72,12 +72,16 @@ "traits.DOB", "context.traits.DOB" ], - "state": ["traits.state", "context.traits.state"], "country": ["traits.country", "context.traits.country"], "region": ["traits.region", "context.traits.region"], "city": ["traits.address.city", "context.traits.address.city"], - + "street": [ + "traits.street", + "traits.address.street", + "context.traits.street", + "context.traits.address.street" + ], "avatar": [ "traits.avatar", "context.traits.avatar", diff --git a/src/v0/util/index.js b/src/v0/util/index.js index ac1bacf404..389b93a7af 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -24,6 +24,8 @@ const { OAuthSecretError, getErrorRespEvents, } = require('@rudderstack/integrations-lib'); + +const { JsonTemplateEngine, PathType } = require('@rudderstack/json-template-engine'); const logger = require('../../logger'); const stats = require('../../util/stats'); const { DestCanonicalNames, DestHandlerMap } = require('../../constants/destinationCanonicalNames'); @@ -57,6 +59,18 @@ const isNull = (x) => lodash.isNull(x); // GENERIC UTLITY // ======================================================================== +const removeUndefinedAndNullRecurse = (obj) => { + // eslint-disable-next-line no-restricted-syntax + for (const key in obj) { + if (obj[key] === null || obj[key] === undefined) { + // eslint-disable-next-line no-param-reassign + delete obj[key]; + } else if (typeof obj[key] === 'object') { + removeUndefinedAndNullRecurse(obj[key]); + } + } +}; + const getEventTime = (message) => { try { return new Date(message.timestamp).toISOString(); @@ -2234,6 +2248,16 @@ const validateEventAndLowerCaseConversion = (event, isMandatory, convertToLowerC return convertToLowerCase ? event.toString().toLowerCase() : event.toString(); }; +const applyCustomMappings = (message, mappings) => { + const flatMappings = mappings.map((mapping) => ({ + input: mapping.from, + output: mapping.to, + })); + return JsonTemplateEngine.createAsSync(flatMappings, { defaultPathType: PathType.JSON }).evaluate( + message, + ); +}; + // ======================================================================== // EXPORTS // ======================================================================== @@ -2242,6 +2266,7 @@ module.exports = { ErrorMessage, addExternalIdToTraits, adduserIdFromExternalId, + applyCustomMappings, base64Convertor, batchMultiplexedEvents, checkEmptyStringInarray, @@ -2318,6 +2343,7 @@ module.exports = { removeUndefinedNullEmptyExclBoolInt, removeUndefinedNullValuesAndEmptyObjectArray, removeUndefinedValues, + removeUndefinedAndNullRecurse, returnArrayOfSubarrays, stripTrailingSlash, toTitleCase, diff --git a/src/v0/util/mapWithJSONPath.js b/src/v0/util/mapWithJSONPath.js new file mode 100644 index 0000000000..7265eb2c85 --- /dev/null +++ b/src/v0/util/mapWithJSONPath.js @@ -0,0 +1,58 @@ +/* eslint-disable no-plusplus */ +const jsonpath = require('rs-jsonpath'); + +function mapWithJsonPath(message, targetObject, sourcePath, targetPath) { + const values = jsonpath.query(message, sourcePath); + const matchTargetPath = targetPath.split('$.events[0].')[1] || targetPath; + const regexMatch = /\[[^\n\]]*]/; + if (regexMatch.test(sourcePath) && regexMatch.test(matchTargetPath)) { + // both paths are arrays + // eslint-disable-next-line unicorn/no-for-loop + for (let i = 0; i < values.length; i++) { + const targetPathWithIndex = targetPath.replace(/\[\*]/g, `[${i}]`); + const tragetValue = values[i] ? values[i] : null; + jsonpath.value(targetObject, targetPathWithIndex, tragetValue); + } + } else if (!regexMatch.test(sourcePath) && regexMatch.test(matchTargetPath)) { + // source path is not array and target path is + const targetPathArr = targetPath.split('.'); + const holdingArr = []; + // eslint-disable-next-line unicorn/no-for-loop + for (let i = 0; i < targetPathArr.length; i++) { + if (/\[\*]/.test(targetPathArr[i])) { + holdingArr.push(targetPathArr[i]); + break; + } else { + holdingArr.push(targetPathArr[i]); + } + } + const parentTargetPath = holdingArr.join('.'); + const exisitngTargetValues = jsonpath.query(targetObject, parentTargetPath); + if (exisitngTargetValues.length > 0) { + for (let i = 0; i < exisitngTargetValues.length; i++) { + const targetPathWithIndex = targetPath.replace(/\[\*]/g, `[${i}]`); + jsonpath.value(targetObject, targetPathWithIndex, values[0]); + } + } else { + const targetPathWithIndex = targetPath.replace(/\[\*]/g, '[0]'); + jsonpath.value(targetObject, targetPathWithIndex, values[0]); + } + } else if (regexMatch.test(sourcePath)) { + // source path is an array but target path is not + + // filter out null values + const filteredValues = values.filter((value) => value !== null); + if (filteredValues.length > 1) { + jsonpath.value(targetObject, targetPath, filteredValues); + } else { + jsonpath.value(targetObject, targetPath, filteredValues[0]); + } + } else { + // both paths are not arrays + jsonpath.value(targetObject, targetPath, values[0]); + } +} + +module.exports = { + mapWithJsonPath, +}; diff --git a/test/integrations/destinations/campaign_manager/processor/data.ts b/test/integrations/destinations/campaign_manager/processor/data.ts index beff44c928..9aa41691c6 100644 --- a/test/integrations/destinations/campaign_manager/processor/data.ts +++ b/test/integrations/destinations/campaign_manager/processor/data.ts @@ -827,4 +827,517 @@ export const data = [ }, }, }, + { + name: 'campaign_manager', + description: 'Test 6: Enhanced Conversions with un-hashed data in request payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + id: '0572f78fa49c648e', + name: 'generic_x86_arm', + type: 'Android', + model: 'AOSP on IA Emulator', + manufacturer: 'Google', + adTrackingEnabled: true, + advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8', + }, + traits: { + email: 'alex@example.com', + phone: '+1-202-555-0146', + firstName: 'John', + lastName: 'Gomes', + city: 'London', + state: 'England', + country: 'GB', + postalCode: 'EC3M', + street: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + }, + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: '34245', + floodlightConfigurationId: '213123123', + ordinal: 'string', + floodlightActivityId: '456543345245', + value: '756', + encryptedUserIdCandidates: ['dfghjbnm'], + quantity: '455678', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + encryptionEntityType: 'DCM_ACCOUNT', + requestType: 'batchupdate', + }, + type: 'track', + event: 'event test', + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + profileId: '5343234', + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + enableEnhancedConversions: true, + isHashingRequired: true, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate', + headers: { + Authorization: 'Bearer dummyApiToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + kind: 'dfareporting#conversionsBatchUpdateRequest', + encryptionInfo: { + encryptionEntityType: 'DCM_ACCOUNT', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + kind: 'dfareporting#encryptionInfo', + }, + conversions: [ + { + floodlightConfigurationId: '213123123', + ordinal: 'string', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 756, + encryptedUserIdCandidates: ['dfghjbnm'], + nonPersonalizedAd: false, + treatmentForUnderage: false, + userIdentifiers: [ + { + hashedEmail: + '6db61e6dcbcf2390e4a46af426f26a133a3bee45021422fc7ae86e9136f14110', + }, + { + hashedPhoneNumber: + 'ec7e6b85f24fa6b796f1017236463f1b7160fbdc5e663e39ab363b6d6fe30b9f', + }, + { + addressInfo: { + hashedFirstName: + '96d9632f363564cc3032521409cf22a852f2032eec099ed5967c0d000cec607a', + hashedLastName: + '12918b23d69d698324a78a8ab8f5060fdb25537ea9620f956d39adca151c3ef9', + hashedStreetAddress: + '5c100d86e9f40bb62a85ca821ff93d96aff6b0dc4c792794c4a4d51ec9246eff', + city: 'London', + state: 'England', + postalCode: 'EC3M', + countryCode: 'GB', + }, + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'campaign_manager', + description: 'Test 7: Enhanced Conversions with hashed data in request payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + id: '0572f78fa49c648e', + name: 'generic_x86_arm', + type: 'Android', + model: 'AOSP on IA Emulator', + manufacturer: 'Google', + adTrackingEnabled: true, + advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8', + }, + traits: { + email: '6db61e6dcbcf2390e4a46af426f26a133a3bee45021422fc7ae86e9136f14110', + phone: '', + firstName: '96d9632f363564cc3032521409cf22a852f2032eec099ed5967c0d000cec607a', + lastName: '12918b23d69d698324a78a8ab8f5060fdb25537ea9620f956d39adca151c3ef9', + city: 'London', + state: 'England', + country: 'GB', + postalCode: 'EC3M', + street: '5c100d86e9f40bb62a85ca821ff93d96aff6b0dc4c792794c4a4d51ec9246eff', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + }, + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: '34245', + floodlightConfigurationId: '213123123', + ordinal: 'string', + floodlightActivityId: '456543345245', + value: '756', + encryptedUserIdCandidates: ['dfghjbnm'], + quantity: '455678', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + encryptionEntityType: 'DCM_ACCOUNT', + requestType: 'batchupdate', + }, + type: 'track', + event: 'event test', + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + profileId: '5343234', + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + enableEnhancedConversions: true, + isHashingRequired: false, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate', + headers: { + Authorization: 'Bearer dummyApiToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + kind: 'dfareporting#conversionsBatchUpdateRequest', + encryptionInfo: { + encryptionEntityType: 'DCM_ACCOUNT', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + kind: 'dfareporting#encryptionInfo', + }, + conversions: [ + { + floodlightConfigurationId: '213123123', + ordinal: 'string', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 756, + encryptedUserIdCandidates: ['dfghjbnm'], + nonPersonalizedAd: false, + treatmentForUnderage: false, + userIdentifiers: [ + { + hashedEmail: + '6db61e6dcbcf2390e4a46af426f26a133a3bee45021422fc7ae86e9136f14110', + }, + { + addressInfo: { + hashedFirstName: + '96d9632f363564cc3032521409cf22a852f2032eec099ed5967c0d000cec607a', + hashedLastName: + '12918b23d69d698324a78a8ab8f5060fdb25537ea9620f956d39adca151c3ef9', + hashedStreetAddress: + '5c100d86e9f40bb62a85ca821ff93d96aff6b0dc4c792794c4a4d51ec9246eff', + city: 'London', + state: 'England', + postalCode: 'EC3M', + countryCode: 'GB', + }, + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'campaign_manager', + description: 'Test 8: Enhanced Conversions with no traits in request payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + id: '0572f78fa49c648e', + name: 'generic_x86_arm', + type: 'Android', + model: 'AOSP on IA Emulator', + manufacturer: 'Google', + adTrackingEnabled: true, + advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + }, + originalTimestamp: '2022-11-17T00:22:02.903+05:30', + properties: { + profileId: '34245', + floodlightConfigurationId: '213123123', + ordinal: 'string', + floodlightActivityId: '456543345245', + value: '756', + encryptedUserIdCandidates: ['dfghjbnm'], + quantity: '455678', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + encryptionEntityType: 'DCM_ACCOUNT', + requestType: 'batchupdate', + }, + type: 'track', + event: 'event test', + anonymousId: 'randomId', + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + profileId: '5343234', + treatmentForUnderage: false, + limitAdTracking: false, + childDirectedTreatment: false, + nonPersonalizedAd: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + enableEnhancedConversions: true, + isHashingRequired: true, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://dfareporting.googleapis.com/dfareporting/v4/userprofiles/34245/conversions/batchupdate', + headers: { + Authorization: 'Bearer dummyApiToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + kind: 'dfareporting#conversionsBatchUpdateRequest', + encryptionInfo: { + encryptionEntityType: 'DCM_ACCOUNT', + encryptionSource: 'AD_SERVING', + encryptionEntityId: '3564523', + kind: 'dfareporting#encryptionInfo', + }, + conversions: [ + { + floodlightConfigurationId: '213123123', + ordinal: 'string', + timestampMicros: '1668624722903000', + floodlightActivityId: '456543345245', + quantity: '455678', + value: 756, + encryptedUserIdCandidates: ['dfghjbnm'], + nonPersonalizedAd: false, + treatmentForUnderage: false, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + secret: { + access_token: 'dummyApiToken', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + statusCode: 200, + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/ga4/processor/data.ts b/test/integrations/destinations/ga4/processor/data.ts index ba5b53c7d2..fb65787214 100644 --- a/test/integrations/destinations/ga4/processor/data.ts +++ b/test/integrations/destinations/ga4/processor/data.ts @@ -1,13 +1,3 @@ -import { pageTestData } from './pageTestData'; -import { ecommTestData } from './ecomTestData'; -import { trackTestData } from './trackTestData'; -import { groupTestData } from './groupTestData'; -import { validationTestData } from './validationTestData'; +import { existingTests } from './exisitngTests'; -export const data = [ - ...pageTestData, - ...trackTestData, - ...ecommTestData, - ...groupTestData, - ...validationTestData, -]; +export const data = [...existingTests]; diff --git a/test/integrations/destinations/ga4/processor/exisitngTests.ts b/test/integrations/destinations/ga4/processor/exisitngTests.ts new file mode 100644 index 0000000000..2913004ca6 --- /dev/null +++ b/test/integrations/destinations/ga4/processor/exisitngTests.ts @@ -0,0 +1,13 @@ +import { pageTestData } from './pageTestData'; +import { ecommTestData } from './ecomTestData'; +import { trackTestData } from './trackTestData'; +import { groupTestData } from './groupTestData'; +import { validationTestData } from './validationTestData'; + +export const existingTests = [ + ...pageTestData, + ...trackTestData, + ...ecommTestData, + ...groupTestData, + ...validationTestData, +]; diff --git a/test/integrations/destinations/ga4_v2/mocks.ts b/test/integrations/destinations/ga4_v2/mocks.ts new file mode 100644 index 0000000000..3a27349ff7 --- /dev/null +++ b/test/integrations/destinations/ga4_v2/mocks.ts @@ -0,0 +1,5 @@ +export const defaultMockFns = () => { + return jest + .spyOn(Date, 'now') + .mockImplementation(() => new Date('2022-04-29T05:17:09Z').valueOf()); +}; diff --git a/test/integrations/destinations/ga4_v2/processor/customMappings.ts b/test/integrations/destinations/ga4_v2/processor/customMappings.ts new file mode 100644 index 0000000000..b1db2121ea --- /dev/null +++ b/test/integrations/destinations/ga4_v2/processor/customMappings.ts @@ -0,0 +1,721 @@ +import { defaultMockFns } from '../mocks'; + +const traits = { + firstName: 'John', + lastName: 'Gomes', + city: 'London', + state: 'UK', + streetAddress: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', + group: 'test group', +}; + +const device = { + adTrackingEnabled: 'true', + advertisingId: 'T0T0T072-5e28-45a1-9eda-ce22a3e36d1a', + id: '3f034872-5e28-45a1-9eda-ce22a3e36d1a', + manufacturer: 'Google', + model: 'AOSP on IA Emulator', + name: 'generic_x86_arm', + type: 'ios', + attTrackingStatus: 3, +}; + +const properties = { + list_id: 'random_list_id', + category: 'random_category', + storePrice: 456, + prices: [ + { + id: 'store-price', + value: 456, + }, + { + id: 'desk-price', + value: 567, + }, + ], + products: [ + { + product_id: 883213, + name: 'Salt', + coupon: 'HHH', + price: 100, + position: 1, + quantity: 10, + affiliation: 'NADA', + currency: 'INR', + discount: '2%', + item_category3: 'grocery', + }, + { + product_id: 213123, + name: 'Sugar', + coupon: 'III', + price: 200, + position: 2, + quantity: 20, + affiliation: 'ADNA', + currency: 'INR', + discount: '5%', + item_category2: 'regulars', + item_category3: 'grocery', + some_data: 'someValue', + }, + ], +}; + +const integrations = { + GA4: { + consents: { + ad_personalization: 'GRANTED', + ad_user_data: 'DENIED', + }, + }, +}; + +const eventsMapping = [ + { + rsEventName: 'Product List Viewed', + destEventName: 'view_item_list', + eventProperties: [ + { + to: '$.client_id', + from: '$.context.traits.anonymousId', + }, + { + to: '$.events[0].params.items[*].name', + from: '$.properties.products[*].name', + }, + { + to: '$.events[0].params.prices', + from: '$.properties.storePrice', + }, + { + to: '$.events[0].params.items[*].id', + from: '$.properties.products[*].product_id', + }, + { + to: '$.events[0].params.items[*].key', + from: '$.properties.products[*].some_data', + }, + { + to: '$.events[0].params.items[*].list_id', + from: '$.properties.list_id', + }, + { + to: '$.userProperties.firstName.value', + from: '$.context.traits.firstName', + }, + { + to: '$.userProperties.lastName.value', + from: '$.context.traits.lastName', + }, + ], + }, + { + rsEventName: 'Product Added', + destEventName: 'add_to_cart', + eventProperties: [ + { + to: '$.client_id', + from: '$.context.traits.anonymousId', + }, + { + to: '$.events[0].params.items[*].name', + from: '$.properties.products[*].name', + }, + { + to: '$.events[0].params.prices', + from: '$.properties.storePrice', + }, + { + to: '$.events[0].params.items[*].id', + from: '$.properties.products[*].product_id', + }, + { + to: '$.events[0].params.items[*].key', + from: '$.properties.products[*].some_data', + }, + { + to: '$.events[0].params.items[*].list_id', + from: '$.properties.list_id', + }, + { + to: '$.userProperties.firstName.value', + from: '$.context.traits.firstName', + }, + { + to: '$.userProperties.lastName.value', + from: '$.context.traits.lastName', + }, + ], + }, + { + rsEventName: 'Product Added', + destEventName: 'checkout_started', + eventProperties: [ + { + to: '$.client_id', + from: '$.context.traits.anonymousId', + }, + { + to: '$.events[0].params.items[*].name', + from: '$.properties.products[*].name', + }, + { + to: '$.events[0].params.prices', + from: '$.properties.storePrice', + }, + { + to: '$.events[0].params.items[*].id', + from: '$.properties.products[*].product_id', + }, + { + to: '$.events[0].params.items[*].key', + from: '$.properties.products[*].some_data', + }, + { + to: '$.events[0].params.items[*].list_id', + from: '$.properties.list_id', + }, + { + to: '$.userProperties.firstName.value', + from: '$.context.traits.firstName', + }, + { + to: '$.userProperties.lastName.value', + from: '$.context.traits.lastName', + }, + ], + }, + { + rsEventName: '$group', + destEventName: 'join_group', + eventProperties: [ + { + to: '$.client_id', + from: '$.context.traits.anonymousId', + }, + { + to: '$.events[0].params.group_id', + from: '$.context.traits.group_id', + }, + { + to: '$.userProperties.firstName.value', + from: '$.context.traits.firstName', + }, + { + to: '$.userProperties.lastName.value', + from: '$.context.traits.lastName', + }, + ], + }, +]; + +const destination = { + Config: { + apiSecret: 'dummyApiSecret', + measurementId: 'G-T40PE6KET4', + firebaseAppId: '', + blockPageViewEvent: false, + typesOfClient: 'gtag', + extendPageViewParams: false, + sendUserId: false, + eventFilteringOption: 'disable', + eventsMapping, + }, +}; +export const customMappingTestCases = [ + { + name: 'ga4_v2', + id: 'ga4_custom_mapping_test_0', + description: 'Custom Mapping Test 0', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'track', + event: 'Product List Viewed', + userId: 'root_user', + anonymousId: 'root_anonId', + context: { + device, + traits, + }, + properties, + originalTimestamp: '2022-04-28T00:23:09.544Z', + integrations, + }, + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.google-analytics.com/mp/collect', + headers: { + HOST: 'www.google-analytics.com', + 'Content-Type': 'application/json', + }, + params: { + api_secret: 'dummyApiSecret', + measurement_id: 'G-T40PE6KET4', + }, + body: { + JSON: { + user_id: 'root_user', + timestamp_micros: 1651105389000000, + non_personalized_ads: false, + client_id: 'root_anonId', + events: [ + { + name: 'view_item_list', + params: { + items: [ + { + name: 'Salt', + id: 883213, + list_id: 'random_list_id', + }, + { + id: 213123, + key: 'someValue', + list_id: 'random_list_id', + name: 'Sugar', + }, + ], + prices: 456, + }, + }, + ], + userProperties: { + firstName: { + value: 'John', + }, + lastName: { + value: 'Gomes', + }, + }, + consent: { + ad_user_data: 'DENIED', + ad_personalization: 'GRANTED', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + name: 'ga4_v2', + id: 'ga4_custom_mapping_test_1', + description: 'Custom Mapping Test for multiplexing', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'track', + event: 'Product Added', + userId: 'root_user', + anonymousId: 'root_anonId', + context: { + device, + traits, + }, + properties, + originalTimestamp: '2022-04-28T00:23:09.544Z', + integrations, + }, + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.google-analytics.com/mp/collect', + headers: { + HOST: 'www.google-analytics.com', + 'Content-Type': 'application/json', + }, + params: { + api_secret: 'dummyApiSecret', + measurement_id: 'G-T40PE6KET4', + }, + body: { + JSON: { + user_id: 'root_user', + timestamp_micros: 1651105389000000, + non_personalized_ads: false, + client_id: 'root_anonId', + events: [ + { + name: 'add_to_cart', + params: { + items: [ + { + name: 'Salt', + id: 883213, + list_id: 'random_list_id', + }, + { + name: 'Sugar', + id: 213123, + key: 'someValue', + list_id: 'random_list_id', + }, + ], + prices: 456, + }, + }, + ], + userProperties: { + firstName: { + value: 'John', + }, + lastName: { + value: 'Gomes', + }, + }, + consent: { + ad_user_data: 'DENIED', + ad_personalization: 'GRANTED', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.google-analytics.com/mp/collect', + headers: { + HOST: 'www.google-analytics.com', + 'Content-Type': 'application/json', + }, + params: { + api_secret: 'dummyApiSecret', + measurement_id: 'G-T40PE6KET4', + }, + body: { + JSON: { + user_id: 'root_user', + timestamp_micros: 1651105389000000, + non_personalized_ads: false, + client_id: 'root_anonId', + events: [ + { + name: 'checkout_started', + params: { + items: [ + { + name: 'Salt', + id: 883213, + list_id: 'random_list_id', + }, + { + name: 'Sugar', + id: 213123, + key: 'someValue', + list_id: 'random_list_id', + }, + ], + prices: 456, + }, + }, + ], + userProperties: { + firstName: { + value: 'John', + }, + lastName: { + value: 'Gomes', + }, + }, + consent: { + ad_user_data: 'DENIED', + ad_personalization: 'GRANTED', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + name: 'ga4_v2', + id: 'ga4_custom_mapping_test_2', + description: 'Custom Mapping Test For mapping not present in events mapping', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'track', + event: 'Product Viewed', + userId: 'root_user', + anonymousId: 'root_anonId', + context: { + device, + traits, + }, + properties, + originalTimestamp: '2022-04-28T00:23:09.544Z', + integrations, + }, + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.google-analytics.com/mp/collect', + headers: { + HOST: 'www.google-analytics.com', + 'Content-Type': 'application/json', + }, + params: { + api_secret: 'dummyApiSecret', + measurement_id: 'G-T40PE6KET4', + }, + body: { + JSON: { + user_id: 'root_user', + timestamp_micros: 1651105389000000, + non_personalized_ads: false, + user_properties: { + firstName: { + value: 'John', + }, + lastName: { + value: 'Gomes', + }, + city: { + value: 'London', + }, + state: { + value: 'UK', + }, + group: { + value: 'test group', + }, + }, + events: [ + { + name: 'Product_Viewed', + params: { + engagement_time_msec: 1, + list_id: 'random_list_id', + category: 'random_category', + storePrice: 456, + prices_0_id: 'store-price', + prices_0_value: 456, + prices_1_id: 'desk-price', + prices_1_value: 567, + products_0_product_id: 883213, + products_0_name: 'Salt', + products_0_coupon: 'HHH', + products_0_price: 100, + products_0_position: 1, + products_0_quantity: 10, + products_0_affiliation: 'NADA', + products_0_currency: 'INR', + products_0_discount: '2%', + products_0_item_category3: 'grocery', + products_1_product_id: 213123, + products_1_name: 'Sugar', + products_1_coupon: 'III', + products_1_price: 200, + products_1_position: 2, + products_1_quantity: 20, + products_1_affiliation: 'ADNA', + products_1_currency: 'INR', + products_1_discount: '5%', + products_1_item_category2: 'regulars', + products_1_item_category3: 'grocery', + products_1_some_data: 'someValue', + }, + }, + ], + consent: { + ad_user_data: 'DENIED', + ad_personalization: 'GRANTED', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + name: 'ga4_v2', + id: 'ga4_custom_mapping_test_3', + description: 'Custom Mapping Test For Group Event Type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'group', + userId: 'root_user', + anonymousId: 'root_anonId', + context: { + device, + traits, + }, + properties, + originalTimestamp: '2022-04-28T00:23:09.544Z', + integrations, + }, + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.google-analytics.com/mp/collect', + headers: { + HOST: 'www.google-analytics.com', + 'Content-Type': 'application/json', + }, + params: { + api_secret: 'dummyApiSecret', + measurement_id: 'G-T40PE6KET4', + }, + body: { + JSON: { + user_id: 'root_user', + timestamp_micros: 1651105389000000, + non_personalized_ads: false, + client_id: 'root_anonId', + events: [ + { + name: 'join_group', + params: { + city: 'London', + engagement_time_msec: 1, + firstName: 'John', + group: 'test group', + lastName: 'Gomes', + state: 'UK', + streetAddress: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', + }, + }, + ], + user_properties: { + firstName: { + value: 'John', + }, + lastName: { + value: 'Gomes', + }, + city: { + value: 'London', + }, + state: { + value: 'UK', + }, + group: { + value: 'test group', + }, + }, + consent: { + ad_user_data: 'DENIED', + ad_personalization: 'GRANTED', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, +]; diff --git a/test/integrations/destinations/ga4_v2/processor/data.ts b/test/integrations/destinations/ga4_v2/processor/data.ts new file mode 100644 index 0000000000..ba82792f31 --- /dev/null +++ b/test/integrations/destinations/ga4_v2/processor/data.ts @@ -0,0 +1,3 @@ +import { customMappingTestCases } from './customMappings'; + +export const data = [...customMappingTestCases]; diff --git a/test/integrations/sources/auth0/data.ts b/test/integrations/sources/auth0/data.ts index b115f0e05e..44b511cad2 100644 --- a/test/integrations/sources/auth0/data.ts +++ b/test/integrations/sources/auth0/data.ts @@ -254,7 +254,6 @@ export const data = [ batch: [ { type: 'identify', - source_type: 'ss', sentAt: '2022-10-31T05:57:06.859Z', traits: { connection: 'Username-Password-Authentication', @@ -291,6 +290,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: 'All Applications', description: '', + source_type: 'ss', }, integrations: { Auth0: false, @@ -305,7 +305,6 @@ export const data = [ batch: [ { type: 'track', - source_type: 'sapi', event: 'Success API Operation', sentAt: '2022-10-31T05:57:06.874Z', userId: 'auth0|dummyPassword', @@ -511,6 +510,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: '', description: 'Create a User', + source_type: 'sapi', }, integrations: { Auth0: false, @@ -596,7 +596,6 @@ export const data = [ batch: [ { type: 'group', - source_type: 'sapi', sentAt: '2022-10-31T06:09:59.135Z', userId: 'google-oauth2|123456', anonymousId: '97fcd7b2-cc24-47d7-b776-057b7b199513', @@ -650,6 +649,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: '', description: 'Add members to an organization', + source_type: 'sapi', }, integrations: { Auth0: false, @@ -939,7 +939,6 @@ export const data = [ batch: [ { type: 'track', - source_type: 'sapi', event: 'Success API Operation', sentAt: '2022-10-31T06:15:25.201Z', userId: 'google-oauth2|123456', @@ -1147,6 +1146,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: '', description: 'Update tenant settings', + source_type: 'sapi', }, integrations: { Auth0: false, @@ -1161,7 +1161,6 @@ export const data = [ batch: [ { type: 'track', - source_type: 'gd_tenant_update', event: 'Guardian tenant update', sentAt: '2022-10-31T06:15:25.196Z', userId: 'google-oauth2|123456', @@ -1221,6 +1220,7 @@ export const data = [ }, }, description: 'Guardian - Updates tenant settings', + source_type: 'gd_tenant_update', }, integrations: { Auth0: false, @@ -1290,7 +1290,6 @@ export const data = [ batch: [ { type: 'identify', - source_type: 'ss', sentAt: '2022-10-31T05:57:06.859Z', traits: { connection: 'Username-Password-Authentication', @@ -1327,6 +1326,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: 'All Applications', description: '', + source_type: 'ss', }, integrations: { Auth0: false, @@ -1410,7 +1410,6 @@ export const data = [ batch: [ { type: 'identify', - source_type: 'ss', userId: '', anonymousId: '97fcd7b2-cc24-47d7-b776-057b7b199513', sentAt: '2022-10-31T05:57:06.859Z', @@ -1447,6 +1446,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: 'All Applications', description: '', + source_type: 'ss', }, integrations: { Auth0: false, @@ -1461,7 +1461,6 @@ export const data = [ batch: [ { type: 'track', - source_type: 'sapi', event: 'Success API Operation', sentAt: '2022-10-31T05:57:06.874Z', anonymousId: '97fcd7b2-cc24-47d7-b776-057b7b199513', @@ -1482,6 +1481,7 @@ export const data = [ client_id: 'vQcJNDTxsM1W72eHFonRJdzyOvawlwIt', client_name: '', description: 'Create a User', + source_type: 'sapi', }, integrations: { Auth0: false,