From 6a3d583d6b4f918cb0fd5c26a25f892642e42e03 Mon Sep 17 00:00:00 2001 From: Dilip Kola Date: Mon, 10 Jun 2024 13:27:44 +0530 Subject: [PATCH] refactor: use json templates for custom mappings updated test cases as some cases are not valid. --- package-lock.json | 17 +++--- package.json | 3 +- .../destinations/ga4/customMappingsHandler.js | 56 +++++++++---------- src/v0/util/index.js | 13 +++++ 4 files changed, 49 insertions(+), 40 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9ab8d24618..4681bd1661 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,8 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.8", - "@rudderstack/workflow-engine": "^0.7.9", + "@rudderstack/json-template-engine": "^0.11.0", + "@rudderstack/workflow-engine": "^0.8.0", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", @@ -4481,17 +4482,17 @@ } }, "node_modules/@rudderstack/json-template-engine": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.10.5.tgz", - "integrity": "sha512-PasCK5RDwiRHsFhAb3w0n+8JPRYcZTffe2l+M/wtzvqU+12NPj3YTEIaMWkhogY6AmPYswAaMX/kr+4j7dKiUA==" + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.11.0.tgz", + "integrity": "sha512-9XrzY7W9mL2lYro2NOSInuDElW7Qk0nP61UbrfJiTQfrzbyaH7ml663eD07a/4ia3uQynITPsSIGHpMgP3qlEw==" }, "node_modules/@rudderstack/workflow-engine": { - "version": "0.7.9", - "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.7.9.tgz", - "integrity": "sha512-uMELZk7UXs40bgQkIk7fIVrfHo/5ld+5I5kYgZt5rcT65H9aNpWjnNRnsKH9dgu+oxiBFAMassZq5ko4hpEdIQ==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.8.0.tgz", + "integrity": "sha512-oBRucBNR29E2PzwHX3hANT0c6V0yFKNMWxDg0jr8Hin4co6KZjxi4FdpkzTNWvk2h+8iXT8NCSPdJBjt03hTrw==", "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", - "@rudderstack/json-template-engine": "^0.10.5", + "@rudderstack/json-template-engine": "^0.11.0", "jsonata": "^2.0.5", "lodash": "^4.17.21", "object-sizeof": "^2.6.4", diff --git a/package.json b/package.json index f62080a8f9..afac6edcb2 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,8 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.8", - "@rudderstack/workflow-engine": "^0.7.9", + "@rudderstack/json-template-engine": "^0.11.0", + "@rudderstack/workflow-engine": "^0.8.0", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", diff --git a/src/v0/destinations/ga4/customMappingsHandler.js b/src/v0/destinations/ga4/customMappingsHandler.js index 6e3e2f1bf8..4750f83bf3 100644 --- a/src/v0/destinations/ga4/customMappingsHandler.js +++ b/src/v0/destinations/ga4/customMappingsHandler.js @@ -23,9 +23,9 @@ const { removeUndefinedAndNullValues, isHybridModeEnabled, getIntegrationsObj, + applyCustomMappings, } = require('../../util'); const { trackCommonConfig, ConfigCategory, mappingConfig } = require('./config'); -const { mapWithJsonPath } = require('../../util/mapWithJSONPath'); const findGA4Events = (eventsMapping, event) => { // Find the event using destructuring and early return @@ -39,7 +39,7 @@ const findGA4Events = (eventsMapping, event) => { return validMappings; }; -const handleCustomMappings = (message, Config) => { +const handleCustomMappings = async (message, Config) => { const { eventsMapping } = Config; let rsEvent = ''; @@ -102,36 +102,35 @@ const handleCustomMappings = (message, Config) => { return buildDeliverablePayload(rawPayload, Config); } - const processedPayloads = validMappings.map((mapping) => { - const eventName = mapping.destEventName; - // reserved event names are not allowed - if (isReservedEventName(eventName)) { - throw new InstrumentationError(`[GA4]:: Reserved event name: ${eventName} are not allowed`); - } - // validation for ga4 event name - validateEventName(eventName); + const processedPayloads = await Promise.all( + validMappings.map(async (mapping) => { + const eventName = mapping.destEventName; + // reserved event names are not allowed + if (isReservedEventName(eventName)) { + throw new InstrumentationError(`[GA4]:: Reserved event name: ${eventName} are not allowed`); + } + // validation for ga4 event name + validateEventName(eventName); - // Add common top level payload - let ga4BasicPayload = constructPayload(message, trackCommonConfig); - ga4BasicPayload = addClientDetails(ga4BasicPayload, message, Config); + // Add common top level payload + let ga4BasicPayload = constructPayload(message, trackCommonConfig); + ga4BasicPayload = addClientDetails(ga4BasicPayload, message, Config); - const eventPropertiesMappings = mapping.eventProperties || {}; + const eventPropertiesMappings = mapping.eventProperties || []; - const ga4MappedPayload = {}; + const ga4MappedPayload = await applyCustomMappings(message, eventPropertiesMappings); - for (const propertyMapping of eventPropertiesMappings) { - mapWithJsonPath(message, ga4MappedPayload, propertyMapping.from, propertyMapping.to); - } - removeUndefinedAndNullRecurse(ga4MappedPayload); + removeUndefinedAndNullRecurse(ga4MappedPayload); - boilerplateOperations(ga4MappedPayload, message, Config, eventName); + boilerplateOperations(ga4MappedPayload, message, Config, eventName); - if (isDefinedAndNotNull(ga4BasicPayload)) { - return { ...ga4BasicPayload, ...ga4MappedPayload }; - } else { - return ga4MappedPayload; - } - }); + if (isDefinedAndNotNull(ga4BasicPayload)) { + return { ...ga4BasicPayload, ...ga4MappedPayload }; + } else { + return ga4MappedPayload; + } + }), + ); return processedPayloads.map((processedPayload) => buildDeliverablePayload(processedPayload, Config), @@ -162,11 +161,6 @@ const boilerplateOperations = (ga4Payload, message, Config, eventName) => { if (!isEmptyObject(consents)) { ga4Payload.consent = consents; } - - // sanitize user properties - if (ga4Payload.user_properties) { - sanitizeUserProperties(ga4Payload.user_properties); - } }; module.exports = { diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 6dd617f217..47489f8d05 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -24,6 +24,8 @@ const { OAuthSecretError, getErrorRespEvents, } = require('@rudderstack/integrations-lib'); + +const { JsonTemplateEngine, PathType } = require('@rudderstack/json-template-engine'); const logger = require('../../logger'); const stats = require('../../util/stats'); const { DestCanonicalNames, DestHandlerMap } = require('../../constants/destinationCanonicalNames'); @@ -2246,6 +2248,16 @@ const validateEventAndLowerCaseConversion = (event, isMandatory, convertToLowerC return convertToLowerCase ? event.toString().toLowerCase() : event.toString(); }; +const applyCustomMappings = (message, mappings) => { + const flatMappings = mappings.map((mapping) => ({ + input: mapping.from, + output: mapping.to, + })); + return JsonTemplateEngine.create(flatMappings, { defaultPathType: PathType.JSON }).evaluate( + message, + ); +}; + // ======================================================================== // EXPORTS // ======================================================================== @@ -2254,6 +2266,7 @@ module.exports = { ErrorMessage, addExternalIdToTraits, adduserIdFromExternalId, + applyCustomMappings, base64Convertor, batchMultiplexedEvents, checkEmptyStringInarray,