From 5bdb63b5d7d1dc2523d1bf7c6c4a97cc7a4d6e1d Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Mon, 6 Nov 2023 22:48:42 +0530 Subject: [PATCH 01/21] feat: geo location enrichment in destination transformations --- src/helpers/geoLocation.ts | 37 +++++++++++++++++++++++++++++ src/middlewares/enricher.ts | 46 +++++++++++++++++++++++++++++++++++++ src/types/index.ts | 15 ++++-------- 3 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 src/helpers/geoLocation.ts create mode 100644 src/middlewares/enricher.ts diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts new file mode 100644 index 0000000000..751bcf0d0d --- /dev/null +++ b/src/helpers/geoLocation.ts @@ -0,0 +1,37 @@ +import cloneDeep from 'lodash/cloneDeep'; +import isEmpty from 'lodash/isEmpty'; +import { FixMe } from '../util/types'; + +export default class GeoLocationHelper { + public static getGeoLocationData(contextObj: Record): Record { + const context = cloneDeep(contextObj || {}); + const isEnrichmentDoneOnContext = !isEmpty(context.geo); + if (!isEnrichmentDoneOnContext) { + // geo-location data was not sent + return {}; + } + const address = context?.address ? cloneDeep(context?.address) : {}; + const addressFieldMapping = { + city: 'city', + country: 'country', + postalCode: 'postal', + state: 'region', + }; + + const mappedAddress = Object.entries(addressFieldMapping).reduce( + (agg, [identifyAddressKey, geoKey]) => { + if (!address?.[identifyAddressKey] && context.geo?.[geoKey]) { + return { [identifyAddressKey]: context.geo[geoKey], ...agg }; + } + return agg; + }, + {}, + ); + + context.address = { + ...context.address, + ...mappedAddress, + }; + return context; + } +} diff --git a/src/middlewares/enricher.ts b/src/middlewares/enricher.ts new file mode 100644 index 0000000000..a6a7ac8c00 --- /dev/null +++ b/src/middlewares/enricher.ts @@ -0,0 +1,46 @@ +import { Context } from "koa"; +import { ProcessorTransformationRequest, RouterTransformationRequest, RouterTransformationRequestData } from "../types"; +import GeoLocationHelper from "../helpers/geoLocation"; + +export type DTRequest = RouterTransformationRequest | ProcessorTransformationRequest[] + +export default class Enricher { + + private static enrichWithGeoInfo(data: RouterTransformationRequestData[]): RouterTransformationRequestData[] { + return data.map(inpEv => { + const geoEnrichedContext = GeoLocationHelper.getGeoLocationData(inpEv.message?.context) + return { + ...inpEv, + message: { + ...inpEv.message, + context: { + ...inpEv.message?.context, + ...geoEnrichedContext + }, + }, + }; + }); + } + + public static enrichGeoLocation (ctx: Context) { + const transformationRequest = ctx.request.body; + let transformationReq: DTRequest; + let reqBody: unknown; + const isRouterTransform = Array.isArray((transformationRequest as RouterTransformationRequest)?.input); + if (isRouterTransform) { + // Router or batch transformation request + transformationReq = transformationRequest as RouterTransformationRequest; + const enrichedEvents: RouterTransformationRequestData[] = Enricher.enrichWithGeoInfo(transformationReq.input); + reqBody = { + input: enrichedEvents, + destType: transformationReq.destType, + }; + } else { + // Processor transformation + transformationReq = transformationRequest as ProcessorTransformationRequest[]; + reqBody = Enricher.enrichWithGeoInfo(transformationReq); + } + ctx.request.body = reqBody; + } + +} \ No newline at end of file diff --git a/src/types/index.ts b/src/types/index.ts index 79efaecb40..09ef33016e 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -83,22 +83,17 @@ type Destination = { type UserTransformationLibrary = { VersionID: string; }; - -type ProcessorTransformationRequest = { - request?: object; - message: object; - metadata: Metadata; - destination: Destination; - libraries: UserTransformationLibrary[]; -}; - type RouterTransformationRequestData = { request?: object; - message: object; + message: RudderMessage; metadata: Metadata; destination: Destination; }; +type ProcessorTransformationRequest = { + libraries: UserTransformationLibrary[]; +} & RouterTransformationRequestData; + type RouterTransformationRequest = { input: RouterTransformationRequestData[]; destType: string; From 0f44408764b9db1425060c2a4551139b6fc86df7 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 7 Nov 2023 15:48:23 +0530 Subject: [PATCH 02/21] chore: add unit-tests for geoLocation logic Signed-off-by: Sai Sankeerth --- src/helpers/geoLocation.ts | 29 +-- src/middlewares/enricher.ts | 41 ++-- src/v0/util/index.js | 38 ++++ test/helper/helper.test.ts | 415 ++++++++++++++++++++++++++++++++++++ 4 files changed, 492 insertions(+), 31 deletions(-) create mode 100644 test/helper/helper.test.ts diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts index 751bcf0d0d..177bc41837 100644 --- a/src/helpers/geoLocation.ts +++ b/src/helpers/geoLocation.ts @@ -1,16 +1,23 @@ import cloneDeep from 'lodash/cloneDeep'; import isEmpty from 'lodash/isEmpty'; +import set from 'set-value'; import { FixMe } from '../util/types'; +import { getKeyAndValueFromMessage } from '../v0/util'; +import * as GenericFieldMappingJson from '../v0/util/data/GenericFieldMapping.json'; export default class GeoLocationHelper { - public static getGeoLocationData(contextObj: Record): Record { - const context = cloneDeep(contextObj || {}); - const isEnrichmentDoneOnContext = !isEmpty(context.geo); - if (!isEnrichmentDoneOnContext) { + public static getGeoLocationData(message: Record): Record { + const msg = cloneDeep(message || {}); + if (isEmpty(msg.context.geo)) { // geo-location data was not sent return {}; } - const address = context?.address ? cloneDeep(context?.address) : {}; + const { value: address, key: foundKey } = + getKeyAndValueFromMessage(message, GenericFieldMappingJson.address) || {}; + let addressKey = foundKey; + if (!foundKey) { + [addressKey] = GenericFieldMappingJson.address; + } const addressFieldMapping = { city: 'city', country: 'country', @@ -20,18 +27,14 @@ export default class GeoLocationHelper { const mappedAddress = Object.entries(addressFieldMapping).reduce( (agg, [identifyAddressKey, geoKey]) => { - if (!address?.[identifyAddressKey] && context.geo?.[geoKey]) { - return { [identifyAddressKey]: context.geo[geoKey], ...agg }; + if (!address?.[identifyAddressKey] && msg?.context?.geo?.[geoKey]) { + return { [identifyAddressKey]: msg.context.geo[geoKey], ...agg }; } return agg; }, {}, ); - - context.address = { - ...context.address, - ...mappedAddress, - }; - return context; + set(msg, addressKey, { ...address, ...mappedAddress }); + return msg; } } diff --git a/src/middlewares/enricher.ts b/src/middlewares/enricher.ts index a6a7ac8c00..eff69a155e 100644 --- a/src/middlewares/enricher.ts +++ b/src/middlewares/enricher.ts @@ -1,36 +1,42 @@ -import { Context } from "koa"; -import { ProcessorTransformationRequest, RouterTransformationRequest, RouterTransformationRequestData } from "../types"; -import GeoLocationHelper from "../helpers/geoLocation"; +import { Context } from 'koa'; +import { + ProcessorTransformationRequest, + RouterTransformationRequest, + RouterTransformationRequestData, +} from '../types'; +import GeoLocationHelper from '../helpers/geoLocation'; -export type DTRequest = RouterTransformationRequest | ProcessorTransformationRequest[] +export type DTRequest = RouterTransformationRequest | ProcessorTransformationRequest[]; export default class Enricher { - - private static enrichWithGeoInfo(data: RouterTransformationRequestData[]): RouterTransformationRequestData[] { - return data.map(inpEv => { - const geoEnrichedContext = GeoLocationHelper.getGeoLocationData(inpEv.message?.context) + private static enrichWithGeoInfo( + data: RouterTransformationRequestData[], + ): RouterTransformationRequestData[] { + return data.map((inpEv) => { + const geoEnrichedMessage = GeoLocationHelper.getGeoLocationData(inpEv.message); return { ...inpEv, - message: { + message: { ...inpEv.message, - context: { - ...inpEv.message?.context, - ...geoEnrichedContext - }, + ...geoEnrichedMessage, }, }; }); } - public static enrichGeoLocation (ctx: Context) { + public static enrichGeoLocation(ctx: Context) { const transformationRequest = ctx.request.body; let transformationReq: DTRequest; let reqBody: unknown; - const isRouterTransform = Array.isArray((transformationRequest as RouterTransformationRequest)?.input); + const isRouterTransform = Array.isArray( + (transformationRequest as RouterTransformationRequest)?.input, + ); if (isRouterTransform) { // Router or batch transformation request transformationReq = transformationRequest as RouterTransformationRequest; - const enrichedEvents: RouterTransformationRequestData[] = Enricher.enrichWithGeoInfo(transformationReq.input); + const enrichedEvents: RouterTransformationRequestData[] = Enricher.enrichWithGeoInfo( + transformationReq.input, + ); reqBody = { input: enrichedEvents, destType: transformationReq.destType, @@ -42,5 +48,4 @@ export default class Enricher { } ctx.request.body = reqBody; } - -} \ No newline at end of file +} diff --git a/src/v0/util/index.js b/src/v0/util/index.js index d6f6621220..93b984d1b1 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -659,6 +659,43 @@ const getValueFromMessage = (message, sourceKeys) => { return null; }; +const getKeyAndValueFromMessage = (message, sourceKeys) => { + if (Array.isArray(sourceKeys) && sourceKeys.length > 0) { + if (sourceKeys.length === 1) { + logger.warn('List with single element is not ideal. Use it as string instead'); + } + // got the possible sourceKeys + // eslint-disable-next-line no-restricted-syntax + for (const sourceKey of sourceKeys) { + let val = null; + // if the sourceKey is an object we expect it to be a operation + if (typeof sourceKey === 'object') { + val = handleSourceKeysOperation({ + message, + operationObject: sourceKey, + }); + } else { + val = get(message, sourceKey); + } + if (val || val === false || val === 0) { + // return only if the value is valid. + // else look for next possible source in precedence + return { value: val, key: sourceKey }; + } + } + } else if (typeof sourceKeys === 'string') { + // got a single key + // - we don't need to iterate over a loop for a single possible value + return { value: get(message, sourceKeys), key: sourceKeys }; + } else { + // wrong sourceKey type. abort + // DEVELOPER ERROR + // TODO - think of a way to crash the pod + throw new PlatformError('Wrong sourceKey type or blank sourceKey array'); + } + return { value: null, key: '' }; +}; + // get a field value from message. // if sourceFromGenericMap is true get its value from GenericFieldMapping.json and use it as sourceKey // else use sourceKey from `data/message.json` for actual field precedence @@ -2192,4 +2229,5 @@ module.exports = { isValidInteger, isNewStatusCodesAccepted, IsGzipSupported, + getKeyAndValueFromMessage, }; diff --git a/test/helper/helper.test.ts b/test/helper/helper.test.ts new file mode 100644 index 0000000000..debf9397a8 --- /dev/null +++ b/test/helper/helper.test.ts @@ -0,0 +1,415 @@ +import GeoLocationHelper from '../../src/helpers/geoLocation'; + +describe('GeoLocationHelper tests', () => { + test('when context.geo is valid object & address is available in context.traits, map all values in context.traits.address', () => { + const contextObj = { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.0-beta.2', + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.0-beta.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + traits: { + email: 'example124@email.com', + name: 'abcd124', + address: { + street: 'dalhousie street', + }, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', + }; + + const msg = { + anonymousId: '297b0750-934b-4411-b66c-9b418cdbc0c9', + channel: 'web', + context: contextObj, + integrations: { + All: true, + }, + messageId: '0bab70e8-bf2f-449a-a19b-ca6e3bfed9b7', + originalTimestamp: '2020-03-23T18:27:28.98Z', + receivedAt: '2020-03-23T23:57:29.022+05:30', + request_ip: '[::1]:51573', + sentAt: '2020-03-23T18:27:28.981Z', + timestamp: '2020-03-23T23:57:29.021+05:30', + type: 'identify', + userId: 'abcd-124', + }; + + const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + + expect(enhancedMsg.context.traits.address).not.toBe(undefined); + expect(enhancedMsg.context.traits.address).toEqual({ + city: 'Gurugram', + country: 'IN', + postalCode: '122001', + state: 'Haryana', + street: 'dalhousie street', + }); + }); + + test('when context.geo is valid object & address is not available at all, map all values in traits.address', () => { + const contextObj = { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.0-beta.2', + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.0-beta.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + traits: { + email: 'example124@email.com', + name: 'abcd124', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', + }; + + const msg = { + anonymousId: '297b0750-934b-4411-b66c-9b418cdbc0c9', + channel: 'web', + context: contextObj, + integrations: { + All: true, + }, + messageId: '0bab70e8-bf2f-449a-a19b-ca6e3bfed9b7', + originalTimestamp: '2020-03-23T18:27:28.98Z', + receivedAt: '2020-03-23T23:57:29.022+05:30', + request_ip: '[::1]:51573', + sentAt: '2020-03-23T18:27:28.981Z', + timestamp: '2020-03-23T23:57:29.021+05:30', + type: 'identify', + userId: 'abcd-124', + }; + + const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + + expect(enhancedMsg.traits.address).not.toBe(undefined); + expect(enhancedMsg.traits.address).toEqual({ + city: 'Gurugram', + country: 'IN', + postalCode: '122001', + state: 'Haryana', + }); + }); + + test('when context.geo is valid object & address has some in traits, enrich those that are not available in traits.address', () => { + const contextObj = { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.0-beta.2', + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.0-beta.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + traits: { + email: 'example124@email.com', + name: 'abcd124', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', + }; + + const msg = { + anonymousId: '297b0750-934b-4411-b66c-9b418cdbc0c9', + channel: 'web', + context: contextObj, + integrations: { + All: true, + }, + traits: { + address: { + state: 'Himachal', + country: 'INDIA', + street: 'damgoo road', + }, + }, + messageId: '0bab70e8-bf2f-449a-a19b-ca6e3bfed9b7', + originalTimestamp: '2020-03-23T18:27:28.98Z', + receivedAt: '2020-03-23T23:57:29.022+05:30', + request_ip: '[::1]:51573', + sentAt: '2020-03-23T18:27:28.981Z', + timestamp: '2020-03-23T23:57:29.021+05:30', + type: 'identify', + userId: 'abcd-124', + }; + + const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + + expect(enhancedMsg.traits.address).not.toBe(undefined); + expect(enhancedMsg.traits.address).toEqual({ + city: 'Gurugram', + postalCode: '122001', + // already in traits.address + state: 'Himachal', + country: 'INDIA', + street: 'damgoo road', + }); + }); + + test('when context.geo is valid object & address is already enhanced, do not enrich with values from context.geo', () => { + const contextObj = { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.0-beta.2', + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.0-beta.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + traits: { + email: 'example124@email.com', + name: 'abcd124', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', + }; + + const msg = { + anonymousId: '297b0750-934b-4411-b66c-9b418cdbc0c9', + channel: 'web', + context: contextObj, + integrations: { + All: true, + }, + traits: { + address: { + state: 'Himachal', + country: 'INDIA', + street: 'damgoo road', + city: 'Dharamshala', + postalCode: '123546', + }, + }, + messageId: '0bab70e8-bf2f-449a-a19b-ca6e3bfed9b7', + originalTimestamp: '2020-03-23T18:27:28.98Z', + receivedAt: '2020-03-23T23:57:29.022+05:30', + request_ip: '[::1]:51573', + sentAt: '2020-03-23T18:27:28.981Z', + timestamp: '2020-03-23T23:57:29.021+05:30', + type: 'identify', + userId: 'abcd-124', + }; + + const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + + expect(enhancedMsg.traits.address).not.toBe(undefined); + expect(enhancedMsg.traits.address).toEqual({ + // already in traits.address + state: 'Himachal', + country: 'INDIA', + street: 'damgoo road', + city: 'Dharamshala', + postalCode: '123546', + }); + }); + + test("when context.geo doesn't have some properties, do not make use of non-available values in context.geo", () => { + const contextObj = { + geo: { + city: '', + country: '', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.0-beta.2', + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.0-beta.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + traits: { + email: 'example124@email.com', + name: 'abcd124', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', + }; + + const msg = { + anonymousId: '297b0750-934b-4411-b66c-9b418cdbc0c9', + channel: 'web', + context: contextObj, + integrations: { + All: true, + }, + traits: { + address: { + state: 'Himachal', + street: 'damgoo road', + postalCode: '123546', + }, + }, + messageId: '0bab70e8-bf2f-449a-a19b-ca6e3bfed9b7', + originalTimestamp: '2020-03-23T18:27:28.98Z', + receivedAt: '2020-03-23T23:57:29.022+05:30', + request_ip: '[::1]:51573', + sentAt: '2020-03-23T18:27:28.981Z', + timestamp: '2020-03-23T23:57:29.021+05:30', + type: 'identify', + userId: 'abcd-124', + }; + + const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + + expect(enhancedMsg.traits.address).not.toBe(undefined); + expect(enhancedMsg.traits.address).toEqual({ + // already in traits.address + state: 'Himachal', + street: 'damgoo road', + postalCode: '123546', + }); + }); + + test("when context.geo doesn't exist, enrichment would not happen", () => { + const contextObj = { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.0-beta.2', + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.0-beta.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + traits: { + email: 'example124@email.com', + name: 'abcd124', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', + }; + + const msg = { + anonymousId: '297b0750-934b-4411-b66c-9b418cdbc0c9', + channel: 'web', + context: contextObj, + integrations: { + All: true, + }, + messageId: '0bab70e8-bf2f-449a-a19b-ca6e3bfed9b7', + originalTimestamp: '2020-03-23T18:27:28.98Z', + receivedAt: '2020-03-23T23:57:29.022+05:30', + request_ip: '[::1]:51573', + sentAt: '2020-03-23T18:27:28.981Z', + timestamp: '2020-03-23T23:57:29.021+05:30', + type: 'identify', + userId: 'abcd-124', + }; + + const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + + expect(enhancedMsg).toEqual({}); + }); +}); From 4bcc8db134833fc41eb43b864daecf087bf18858 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 7 Nov 2023 15:52:40 +0530 Subject: [PATCH 03/21] chore: provide flexibility for an optional rudderMessage schema Signed-off-by: Sai Sankeerth --- src/types/index.ts | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/types/index.ts b/src/types/index.ts index 09ef33016e..44b8b407b2 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -83,9 +83,37 @@ type Destination = { type UserTransformationLibrary = { VersionID: string; }; + +type RudderTimestamp = Date | string; +type RudderMessageType = + | 'identify' + | 'track' + | 'page' + | 'screen' + | 'group' + | 'alias' + | 'audiencelist'; + +type RudderMessageSchema = Partial<{ + userId: string; + anonymousId: string; + type: RudderMessageType; + channel: string; + context: object; + originalTimestamp: RudderTimestamp; + sentAt: RudderTimestamp; + timestamp: RudderTimestamp; + event: string; + integrations: object; + messageId: string; + properties: object; + traits: object; + [key: string]: FixMe; +}>; + type RouterTransformationRequestData = { request?: object; - message: RudderMessage; + message: RudderMessageSchema; metadata: Metadata; destination: Destination; }; @@ -241,4 +269,5 @@ export { UserDeletionResponse, Destination, ComparatorInput, + RudderMessageSchema, }; From d789f0eb911efbf5d559cac6eff84a3f076d3e1d Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 7 Nov 2023 17:15:31 +0530 Subject: [PATCH 04/21] feat: add tests for unit-testing enrichment logic Signed-off-by: Sai Sankeerth --- test/middleware/enricher.test.ts | 645 +++++++++++++++++++++++++++++++ 1 file changed, 645 insertions(+) create mode 100644 test/middleware/enricher.test.ts diff --git a/test/middleware/enricher.test.ts b/test/middleware/enricher.test.ts new file mode 100644 index 0000000000..3b173800f3 --- /dev/null +++ b/test/middleware/enricher.test.ts @@ -0,0 +1,645 @@ +import { ProcessorTransformationRequest } from '../../src/types'; +import Enricher from '../../src/middlewares/enricher'; + +describe('geoLocation Enrichment during processor transformation tests', () => { + test('should enrich when context.geo is populated correctly', () => { + const inputData: ProcessorTransformationRequest[] = [ + { + destination: { + ID: '1afjtc6chkhdeKsXYrNFOzR5D9v', + Name: 'Autopilot', + DestinationDefinition: { + ID: '1afjX4MlAucK57Q0ctTVlD27Tvo', + Name: 'AUTOPILOT', + DisplayName: 'Autopilot', + Config: { + cdkEnabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + customMappings: [ + { + from: '0001', + to: 'Signup', + }, + ], + triggerId: '00XX', + }, + Enabled: true, + Transformations: [], + // @ts-ignore + IsProcessorEnabled: true, + }, + message: { + anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', + channel: 'web', + context: { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.2', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + integrations: { + All: true, + }, + messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + traits: { + age: 23, + email: 'testmp@rudderstack.com', + firstname: 'Test Kafka', + }, + timestamp: '2020-04-17T20:12:44.758+05:30', + type: 'identify', + userId: 'user12345', + }, + }, + ]; + const ctx = { request: { body: inputData } }; + // @ts-ignore + Enricher.enrichGeoLocation(ctx); + expect(ctx.request.body[0].message.traits).toMatchObject( + expect.objectContaining({ + age: 23, + email: 'testmp@rudderstack.com', + firstname: 'Test Kafka', + address: { + city: 'Gurugram', + country: 'IN', + postalCode: '122001', + state: 'Haryana', + }, + }), + ); + }); + + test('should not enrich when address is already enhanced', () => { + const inputData: ProcessorTransformationRequest[] = [ + { + destination: { + ID: '1afjtc6chkhdeKsXYrNFOzR5D9v', + Name: 'Autopilot', + DestinationDefinition: { + ID: '1afjX4MlAucK57Q0ctTVlD27Tvo', + Name: 'AUTOPILOT', + DisplayName: 'Autopilot', + Config: { + cdkEnabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + customMappings: [ + { + from: '0001', + to: 'Signup', + }, + ], + triggerId: '00XX', + }, + Enabled: true, + Transformations: [], + // @ts-ignore + IsProcessorEnabled: true, + }, + message: { + anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', + channel: 'web', + context: { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.2', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + integrations: { + All: true, + }, + messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + traits: { + age: 23, + email: 'testmp1@rudderstack.com', + firstname: 'Test Kafka2', + address: { + state: 'Himachal', + country: 'INDIA', + street: 'damgoo road', + postalCode: '123321', + city: 'Bandarpur', + }, + }, + timestamp: '2020-04-17T20:12:44.758+05:30', + type: 'identify', + userId: 'user12345', + }, + }, + ]; + const ctx = { request: { body: inputData } }; + // @ts-ignore + Enricher.enrichGeoLocation(ctx); + expect(ctx.request.body[0].message.traits).toMatchObject( + expect.objectContaining({ + age: 23, + email: 'testmp1@rudderstack.com', + firstname: 'Test Kafka2', + address: { + state: 'Himachal', + country: 'INDIA', + street: 'damgoo road', + postalCode: '123321', + city: 'Bandarpur', + }, + }), + ); + }); + + test('should enrich only those fields that are not already enriched when address already contains partial data', () => { + const inputData: ProcessorTransformationRequest[] = [ + { + destination: { + ID: '1afjtc6chkhdeKsXYrNFOzR5D9v', + Name: 'Autopilot', + DestinationDefinition: { + ID: '1afjX4MlAucK57Q0ctTVlD27Tvo', + Name: 'AUTOPILOT', + DisplayName: 'Autopilot', + Config: { + cdkEnabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + customMappings: [ + { + from: '0001', + to: 'Signup', + }, + ], + triggerId: '00XX', + }, + Enabled: true, + Transformations: [], + // @ts-ignore + IsProcessorEnabled: true, + }, + message: { + anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', + channel: 'web', + context: { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.2', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + integrations: { + All: true, + }, + messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + traits: { + age: 23, + email: 'testmp1@rudderstack.com', + firstname: 'Test Kafka2', + address: { + state: 'Himachal', + country: 'INDIA', + street: 'damgoo road', + postalCode: '123321', + }, + }, + timestamp: '2020-04-17T20:12:44.758+05:30', + type: 'identify', + userId: 'user12345', + }, + }, + ]; + const ctx = { request: { body: inputData } }; + // @ts-ignore + Enricher.enrichGeoLocation(ctx); + expect(ctx.request.body[0].message.traits).toMatchObject( + expect.objectContaining({ + age: 23, + email: 'testmp1@rudderstack.com', + firstname: 'Test Kafka2', + address: { + state: 'Himachal', + country: 'INDIA', + street: 'damgoo road', + postalCode: '123321', + // enriched field + city: 'Gurugram', + }, + }), + ); + }); + + test('should not enrich when context.geo is not populated', () => { + const inputData: ProcessorTransformationRequest[] = [ + { + destination: { + ID: '1afjtc6chkhdeKsXYrNFOzR5D9v', + Name: 'Autopilot', + DestinationDefinition: { + ID: '1afjX4MlAucK57Q0ctTVlD27Tvo', + Name: 'AUTOPILOT', + DisplayName: 'Autopilot', + Config: { + cdkEnabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + customMappings: [ + { + from: '0001', + to: 'Signup', + }, + ], + triggerId: '00XX', + }, + Enabled: true, + Transformations: [], + // @ts-ignore + IsProcessorEnabled: true, + }, + message: { + anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.2', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + integrations: { + All: true, + }, + messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + traits: { + age: 23, + email: 'testmp@rudderstack.com', + firstname: 'Test Kafka', + }, + timestamp: '2020-04-17T20:12:44.758+05:30', + type: 'identify', + userId: 'user12345', + }, + }, + ]; + const ctx = { request: { body: inputData } }; + // @ts-ignore + Enricher.enrichGeoLocation(ctx); + + // @ts-ignore + expect(ctx.request.body[0].message.traits?.address).toBe(undefined); + }); + + test('should enrich when context.geo is populated correctly for multiple payloads with their own geolocation data', () => { + const inputData: ProcessorTransformationRequest[] = [ + { + destination: { + ID: '1afjtc6chkhdeKsXYrNFOzR5D9v', + Name: 'Autopilot', + DestinationDefinition: { + ID: '1afjX4MlAucK57Q0ctTVlD27Tvo', + Name: 'AUTOPILOT', + DisplayName: 'Autopilot', + Config: { + cdkEnabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + customMappings: [ + { + from: '0001', + to: 'Signup', + }, + ], + triggerId: '00XX', + }, + Enabled: true, + Transformations: [], + // @ts-ignore + IsProcessorEnabled: true, + }, + message: { + anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', + channel: 'web', + context: { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.2', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + integrations: { + All: true, + }, + messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + traits: { + age: 23, + email: 'testmp@rudderstack.com', + firstname: 'Test Kafka', + }, + timestamp: '2020-04-17T20:12:44.758+05:30', + type: 'identify', + userId: 'user12345', + }, + }, + { + destination: { + ID: '1afjtc6chkhdeKsXYrNFOzR5D9v', + Name: 'Autopilot', + DestinationDefinition: { + ID: '1afjX4MlAucK57Q0ctTVlD27Tvo', + Name: 'AUTOPILOT', + DisplayName: 'Autopilot', + Config: { + cdkEnabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + customMappings: [ + { + from: '0001', + to: 'Signup', + }, + ], + triggerId: '00XX', + }, + Enabled: true, + Transformations: [], + // @ts-ignore + IsProcessorEnabled: true, + }, + message: { + anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', + channel: 'web', + context: { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122002', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.2', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + integrations: { + All: true, + }, + messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + traits: { + address: { + street: 'janamsthan', + }, + }, + timestamp: '2020-04-17T20:12:44.758+05:30', + type: 'identify', + userId: 'user12345', + }, + }, + ]; + const ctx = { request: { body: inputData } }; + // @ts-ignore + Enricher.enrichGeoLocation(ctx); + expect(ctx.request.body[0].message.traits).toMatchObject( + expect.objectContaining({ + age: 23, + email: 'testmp@rudderstack.com', + firstname: 'Test Kafka', + address: { + city: 'Gurugram', + country: 'IN', + postalCode: '122001', + state: 'Haryana', + }, + }), + ); + expect(ctx.request.body[1].message.traits).toMatchObject( + expect.objectContaining({ + address: { + city: 'Gurugram', + country: 'IN', + postalCode: '122002', + state: 'Haryana', + street: 'janamsthan', + }, + }), + ); + }); +}); From d3d6b7281276f01c29daafff928623c73f11e2c2 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 7 Nov 2023 18:40:43 +0530 Subject: [PATCH 05/21] fix: address and context.geo not present case Signed-off-by: Sai Sankeerth --- src/helpers/geoLocation.ts | 4 +- test/middleware/enricher.test.ts | 170 ++++++++++++++++++++++++++++++- 2 files changed, 171 insertions(+), 3 deletions(-) diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts index 177bc41837..1c9dfe6d62 100644 --- a/src/helpers/geoLocation.ts +++ b/src/helpers/geoLocation.ts @@ -34,7 +34,9 @@ export default class GeoLocationHelper { }, {}, ); - set(msg, addressKey, { ...address, ...mappedAddress }); + if (!isEmpty(address) || !isEmpty(mappedAddress)) { + set(msg, addressKey, { ...address, ...mappedAddress }); + } return msg; } } diff --git a/test/middleware/enricher.test.ts b/test/middleware/enricher.test.ts index 3b173800f3..5474ae3cfb 100644 --- a/test/middleware/enricher.test.ts +++ b/test/middleware/enricher.test.ts @@ -1,7 +1,7 @@ -import { ProcessorTransformationRequest } from '../../src/types'; +import { ProcessorTransformationRequest, RouterTransformationRequest } from '../../src/types'; import Enricher from '../../src/middlewares/enricher'; -describe('geoLocation Enrichment during processor transformation tests', () => { +describe('[GeoLocation Enrichment] Processor transformation tests', () => { test('should enrich when context.geo is populated correctly', () => { const inputData: ProcessorTransformationRequest[] = [ { @@ -435,6 +435,172 @@ describe('geoLocation Enrichment during processor transformation tests', () => { expect(ctx.request.body[0].message.traits?.address).toBe(undefined); }); + test('should not contain address object, when context.geo & address are not present', () => { + const inputData: ProcessorTransformationRequest[] = [ + { + destination: { + ID: '1afjtc6chkhdeKsXYrNFOzR5D9v', + Name: 'Autopilot', + DestinationDefinition: { + ID: '1afjX4MlAucK57Q0ctTVlD27Tvo', + Name: 'AUTOPILOT', + DisplayName: 'Autopilot', + Config: { + cdkEnabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + customMappings: [ + { + from: '0001', + to: 'Signup', + }, + ], + triggerId: '00XX', + }, + Enabled: true, + Transformations: [], + // @ts-ignore + IsProcessorEnabled: true, + }, + message: { + anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.2', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + integrations: { + All: true, + }, + messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + traits: { + age: 23, + email: 'testmp@rudderstack.com', + firstname: 'Test Kafka', + }, + timestamp: '2020-04-17T20:12:44.758+05:30', + type: 'identify', + userId: 'user12345', + }, + }, + { + destination: { + ID: '1afjtc6chkhdeKsXYrNFOzR5D9v', + Name: 'Autopilot', + DestinationDefinition: { + ID: '1afjX4MlAucK57Q0ctTVlD27Tvo', + Name: 'AUTOPILOT', + DisplayName: 'Autopilot', + Config: { + cdkEnabled: true, + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + customMappings: [ + { + from: '0001', + to: 'Signup', + }, + ], + triggerId: '00XX', + }, + Enabled: true, + Transformations: [], + // @ts-ignore + IsProcessorEnabled: true, + }, + message: { + anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.2', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + integrations: { + All: true, + }, + messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + traits: { + key: 'val', + }, + timestamp: '2020-04-17T20:12:44.758+05:30', + type: 'identify', + userId: 'user12345', + }, + }, + ]; + const ctx = { request: { body: inputData } }; + // @ts-ignore + Enricher.enrichGeoLocation(ctx); + expect(ctx.request.body[0].message.traits).not.toHaveProperty('address'); + expect(ctx.request.body[1].message.traits).not.toHaveProperty('address'); + }); + test('should enrich when context.geo is populated correctly for multiple payloads with their own geolocation data', () => { const inputData: ProcessorTransformationRequest[] = [ { From 224944105f00a98a2246495e53bbd4ccf3f75316 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 7 Nov 2023 22:34:32 +0530 Subject: [PATCH 06/21] fix: logic to include geolocation info in first found key - add router transformation test-case - change test-case for helper for first-key found logic Signed-off-by: Sai Sankeerth --- src/helpers/geoLocation.ts | 24 +++- test/helper/helper.test.ts | 4 +- test/middleware/enricher.test.ts | 216 +++++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+), 8 deletions(-) diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts index 1c9dfe6d62..1a9b6a3939 100644 --- a/src/helpers/geoLocation.ts +++ b/src/helpers/geoLocation.ts @@ -6,18 +6,30 @@ import { getKeyAndValueFromMessage } from '../v0/util'; import * as GenericFieldMappingJson from '../v0/util/data/GenericFieldMapping.json'; export default class GeoLocationHelper { + public static getAddressKeyAndValue(message: Record): { + addressKey: string; + address: Record; + } { + const { value: address, key: foundKey } = + getKeyAndValueFromMessage(message, GenericFieldMappingJson.address) || {}; + const { key: traitsKey } = getKeyAndValueFromMessage(message, GenericFieldMappingJson.traits); + let addressKey = foundKey; + if (!foundKey) { + addressKey = `${traitsKey}.address`; + if (!traitsKey) { + [addressKey] = GenericFieldMappingJson.address; + } + } + return { addressKey, address }; + } + public static getGeoLocationData(message: Record): Record { const msg = cloneDeep(message || {}); if (isEmpty(msg.context.geo)) { // geo-location data was not sent return {}; } - const { value: address, key: foundKey } = - getKeyAndValueFromMessage(message, GenericFieldMappingJson.address) || {}; - let addressKey = foundKey; - if (!foundKey) { - [addressKey] = GenericFieldMappingJson.address; - } + const { address, addressKey } = GeoLocationHelper.getAddressKeyAndValue(message); const addressFieldMapping = { city: 'city', country: 'country', diff --git a/test/helper/helper.test.ts b/test/helper/helper.test.ts index debf9397a8..cad6cabee6 100644 --- a/test/helper/helper.test.ts +++ b/test/helper/helper.test.ts @@ -128,8 +128,8 @@ describe('GeoLocationHelper tests', () => { const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); - expect(enhancedMsg.traits.address).not.toBe(undefined); - expect(enhancedMsg.traits.address).toEqual({ + expect(enhancedMsg.context.traits.address).not.toBe(undefined); + expect(enhancedMsg.context.traits.address).toEqual({ city: 'Gurugram', country: 'IN', postalCode: '122001', diff --git a/test/middleware/enricher.test.ts b/test/middleware/enricher.test.ts index 5474ae3cfb..cdf2a33cd1 100644 --- a/test/middleware/enricher.test.ts +++ b/test/middleware/enricher.test.ts @@ -809,3 +809,219 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ); }); }); + +describe('[GeoLocation Enrichment] Router/Batch transformation tests', () => { + test('should enrich with geo information when context.geo is present', () => { + const inputData: RouterTransformationRequest = { + input: [ + { + destination: { + ID: '1afjtc6chkhdeKsXYrNFOzR5D9v', + Name: 'Autopilot', + DestinationDefinition: { + ID: '1afjX4MlAucK57Q0ctTVlD27Tvo', + Name: 'AUTOPILOT', + DisplayName: 'Autopilot', + Config: { + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + customMappings: [ + { + from: '0001', + to: 'Signup', + }, + ], + triggerId: '00XX', + }, + Enabled: true, + Transformations: [], + // @ts-ignore + IsProcessorEnabled: true, + }, + message: { + anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', + channel: 'web', + context: { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.2', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + integrations: { + All: true, + }, + messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + traits: { + age: 23, + email: 'testm3p@rudderstack.com', + firstname: 'Test Kafka', + }, + timestamp: '2020-04-17T20:12:44.758+05:30', + type: 'identify', + userId: 'user12345', + }, + }, + { + destination: { + ID: '1afjtc6chkhdeKsXYrNFOzR5D9v', + Name: 'Autopilot', + DestinationDefinition: { + ID: '1afjX4MlAucK57Q0ctTVlD27Tvo', + Name: 'AUTOPILOT', + DisplayName: 'Autopilot', + Config: { + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + apiKey: 'dummyApiKey', + customMappings: [ + { + from: '0001', + to: 'Signup', + }, + ], + triggerId: '00XX', + }, + Enabled: true, + Transformations: [], + // @ts-ignore + IsProcessorEnabled: true, + }, + message: { + anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', + channel: 'web', + context: { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122002', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.2', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.2', + }, + locale: 'en-GB', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + traits: { + age: 23, + email: 'testmp@rudderstack.com', + firstname: 'Test Kafka', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + event: 'test track with property', + integrations: { + All: true, + }, + messageId: '37b75e61-9bd2-4fb8-91ed-e3a064905f3a', + originalTimestamp: '2020-04-17T14:42:44.724Z', + properties: { + test_prop_1: 'test prop', + test_prop_2: 1232, + }, + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53512', + sentAt: '2020-04-17T14:42:44.725Z', + timestamp: '2020-04-17T20:12:44.757+05:30', + type: 'track', + userId: 'user12345', + }, + }, + ], + destType: 'autopilot', + }; + const ctx = { request: { body: inputData } }; + // @ts-ignore + Enricher.enrichGeoLocation(ctx); + expect(ctx.request.body.input[0].message.traits).toMatchObject( + expect.objectContaining({ + age: 23, + email: 'testm3p@rudderstack.com', + firstname: 'Test Kafka', + address: { + city: 'Gurugram', + country: 'IN', + postalCode: '122001', + state: 'Haryana', + }, + }), + ); + // @ts-ignore + expect(ctx.request.body.input[1].message.context?.traits).toMatchObject( + expect.objectContaining({ + address: { + city: 'Gurugram', + country: 'IN', + postalCode: '122002', + state: 'Haryana', + }, + }), + ); + }); +}); From 6dd5c8b6ef256166c518a441cb3d5fb03af47fe4 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 7 Nov 2023 22:51:33 +0530 Subject: [PATCH 07/21] fix: add unit-tests for getting addressKey Signed-off-by: Sai Sankeerth --- test/helper/helper.test.ts | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/test/helper/helper.test.ts b/test/helper/helper.test.ts index cad6cabee6..1677f1e5b7 100644 --- a/test/helper/helper.test.ts +++ b/test/helper/helper.test.ts @@ -413,3 +413,93 @@ describe('GeoLocationHelper tests', () => { expect(enhancedMsg).toEqual({}); }); }); + +describe('get addressKey & address tests', () => { + test('addressKey should be "traits.address", when address is not present but traits object is present', () => { + const { addressKey } = GeoLocationHelper.getAddressKeyAndValue({ + traits: { + name: 'Bruce Wayne', + age: 35, + title: 'The Dark Knight', + }, + }); + expect(addressKey).toEqual('traits.address'); + }); + + test('addressKey should be "context.traits.address", when address is not present but traits object is present', () => { + const { addressKey } = GeoLocationHelper.getAddressKeyAndValue({ + context: { + traits: { + name: 'Bruce Wayne', + age: 35, + title: 'The Dark Knight', + }, + }, + }); + expect(addressKey).toEqual('context.traits.address'); + }); + + test('addressKey should be "traits.address", when traits object is not present at all', () => { + const { addressKey } = GeoLocationHelper.getAddressKeyAndValue({ + anonymousId: '129893-2idi9292', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + context: { + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + }); + expect(addressKey).toEqual('traits.address'); + }); + + test('addressKey should be "context.traits.address", when address is present in context.traits & address is not present in traits', () => { + const { addressKey } = GeoLocationHelper.getAddressKeyAndValue({ + anonymousId: '129893-2idi9292', + originalTimestamp: '2020-04-17T14:42:44.722Z', + receivedAt: '2020-04-17T20:12:44.758+05:30', + request_ip: '[::1]:53513', + sentAt: '2020-04-17T14:42:44.722Z', + context: { + page: { + path: '/tests/html/index4.html', + referrer: '', + search: '', + title: '', + url: 'http://localhost/tests/html/index4.html', + }, + screen: { + density: 2, + }, + traits: { + address: { + city: 'Gotham', + }, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', + }, + traits: { + age: 23, + firstname: 'Selina Kyle', + }, + }); + expect(addressKey).toEqual('context.traits.address'); + }); + + test('addressKey should be "traits.address", when empty payload is sent', () => { + const { addressKey } = GeoLocationHelper.getAddressKeyAndValue({}); + expect(addressKey).toEqual('traits.address'); + }); +}); From 53fdb76d262cf4137799da6ef9581a6f5df82645 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 7 Nov 2023 22:58:16 +0530 Subject: [PATCH 08/21] fix: remove unnecessary operation for getKeyAndValueFromMessage Signed-off-by: Sai Sankeerth --- src/v0/util/index.js | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 93b984d1b1..d53102799e 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -667,16 +667,7 @@ const getKeyAndValueFromMessage = (message, sourceKeys) => { // got the possible sourceKeys // eslint-disable-next-line no-restricted-syntax for (const sourceKey of sourceKeys) { - let val = null; - // if the sourceKey is an object we expect it to be a operation - if (typeof sourceKey === 'object') { - val = handleSourceKeysOperation({ - message, - operationObject: sourceKey, - }); - } else { - val = get(message, sourceKey); - } + const val = get(message, sourceKey); if (val || val === false || val === 0) { // return only if the value is valid. // else look for next possible source in precedence From 8fb76681ccbfa20934e3cab3ab52185b3ecbdec5 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Wed, 8 Nov 2023 10:50:44 +0530 Subject: [PATCH 09/21] chore: remove unnecessary logic in getKeyAndValueFromMessage Signed-off-by: Sai Sankeerth --- src/v0/util/index.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/v0/util/index.js b/src/v0/util/index.js index d53102799e..15412d0b5f 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -660,10 +660,7 @@ const getValueFromMessage = (message, sourceKeys) => { }; const getKeyAndValueFromMessage = (message, sourceKeys) => { - if (Array.isArray(sourceKeys) && sourceKeys.length > 0) { - if (sourceKeys.length === 1) { - logger.warn('List with single element is not ideal. Use it as string instead'); - } + if (Array.isArray(sourceKeys)) { // got the possible sourceKeys // eslint-disable-next-line no-restricted-syntax for (const sourceKey of sourceKeys) { @@ -678,11 +675,6 @@ const getKeyAndValueFromMessage = (message, sourceKeys) => { // got a single key // - we don't need to iterate over a loop for a single possible value return { value: get(message, sourceKeys), key: sourceKeys }; - } else { - // wrong sourceKey type. abort - // DEVELOPER ERROR - // TODO - think of a way to crash the pod - throw new PlatformError('Wrong sourceKey type or blank sourceKey array'); } return { value: null, key: '' }; }; From c2363d0365a357b68e3b99cc3232808430fe424c Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Wed, 8 Nov 2023 19:22:39 +0530 Subject: [PATCH 10/21] feat: add geo location middleware to destination transformation endpoints & asynchronise geolocation enricher Signed-off-by: Sai Sankeerth --- src/helpers/geoLocation.ts | 2 +- src/middlewares/enricher.ts | 5 +++-- src/routes/destination.ts | 4 ++++ test/middleware/enricher.test.ts | 28 ++++++++++++++-------------- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts index 1a9b6a3939..710c02c5ef 100644 --- a/src/helpers/geoLocation.ts +++ b/src/helpers/geoLocation.ts @@ -25,7 +25,7 @@ export default class GeoLocationHelper { public static getGeoLocationData(message: Record): Record { const msg = cloneDeep(message || {}); - if (isEmpty(msg.context.geo)) { + if (isEmpty(msg?.context?.geo || {})) { // geo-location data was not sent return {}; } diff --git a/src/middlewares/enricher.ts b/src/middlewares/enricher.ts index eff69a155e..0b84b19ab0 100644 --- a/src/middlewares/enricher.ts +++ b/src/middlewares/enricher.ts @@ -1,4 +1,4 @@ -import { Context } from 'koa'; +import { Context, Next } from 'koa'; import { ProcessorTransformationRequest, RouterTransformationRequest, @@ -24,7 +24,7 @@ export default class Enricher { }); } - public static enrichGeoLocation(ctx: Context) { + public static async enrichGeoLocation(ctx: Context, next: Next) { const transformationRequest = ctx.request.body; let transformationReq: DTRequest; let reqBody: unknown; @@ -47,5 +47,6 @@ export default class Enricher { reqBody = Enricher.enrichWithGeoInfo(transformationReq); } ctx.request.body = reqBody; + await next(); } } diff --git a/src/routes/destination.ts b/src/routes/destination.ts index 3d4be42ff3..e8caaf99e7 100644 --- a/src/routes/destination.ts +++ b/src/routes/destination.ts @@ -3,6 +3,7 @@ import DestinationController from '../controllers/destination'; import RegulationController from '../controllers/regulation'; import FeatureFlagController from '../middlewares/featureFlag'; import RouteActivationController from '../middlewares/routeActivation'; +import Enricher from '../middlewares/enricher'; const router = new Router(); @@ -11,6 +12,7 @@ router.post( RouteActivationController.isDestinationRouteActive, RouteActivationController.destinationProcFilter, FeatureFlagController.handle, + Enricher.enrichGeoLocation, DestinationController.destinationTransformAtProcessor, ); router.post( @@ -18,6 +20,7 @@ router.post( RouteActivationController.isDestinationRouteActive, RouteActivationController.destinationRtFilter, FeatureFlagController.handle, + Enricher.enrichGeoLocation, DestinationController.destinationTransformAtRouter, ); router.post( @@ -25,6 +28,7 @@ router.post( RouteActivationController.isDestinationRouteActive, RouteActivationController.destinationBatchFilter, FeatureFlagController.handle, + Enricher.enrichGeoLocation, DestinationController.batchProcess, ); diff --git a/test/middleware/enricher.test.ts b/test/middleware/enricher.test.ts index cdf2a33cd1..23fb3ee924 100644 --- a/test/middleware/enricher.test.ts +++ b/test/middleware/enricher.test.ts @@ -2,7 +2,7 @@ import { ProcessorTransformationRequest, RouterTransformationRequest } from '../ import Enricher from '../../src/middlewares/enricher'; describe('[GeoLocation Enrichment] Processor transformation tests', () => { - test('should enrich when context.geo is populated correctly', () => { + test('should enrich when context.geo is populated correctly', async () => { const inputData: ProcessorTransformationRequest[] = [ { destination: { @@ -95,7 +95,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - Enricher.enrichGeoLocation(ctx); + await Enricher.enrichGeoLocation(ctx, () => {}); expect(ctx.request.body[0].message.traits).toMatchObject( expect.objectContaining({ age: 23, @@ -111,7 +111,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ); }); - test('should not enrich when address is already enhanced', () => { + test('should not enrich when address is already enhanced', async () => { const inputData: ProcessorTransformationRequest[] = [ { destination: { @@ -211,7 +211,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - Enricher.enrichGeoLocation(ctx); + await Enricher.enrichGeoLocation(ctx, () => {}); expect(ctx.request.body[0].message.traits).toMatchObject( expect.objectContaining({ age: 23, @@ -228,7 +228,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ); }); - test('should enrich only those fields that are not already enriched when address already contains partial data', () => { + test('should enrich only those fields that are not already enriched when address already contains partial data', async () => { const inputData: ProcessorTransformationRequest[] = [ { destination: { @@ -327,7 +327,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - Enricher.enrichGeoLocation(ctx); + await Enricher.enrichGeoLocation(ctx, () => {}); expect(ctx.request.body[0].message.traits).toMatchObject( expect.objectContaining({ age: 23, @@ -345,7 +345,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ); }); - test('should not enrich when context.geo is not populated', () => { + test('should not enrich when context.geo is not populated', async () => { const inputData: ProcessorTransformationRequest[] = [ { destination: { @@ -429,13 +429,13 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - Enricher.enrichGeoLocation(ctx); + await Enricher.enrichGeoLocation(ctx, () => {}); // @ts-ignore expect(ctx.request.body[0].message.traits?.address).toBe(undefined); }); - test('should not contain address object, when context.geo & address are not present', () => { + test('should not contain address object, when context.geo & address are not present', async () => { const inputData: ProcessorTransformationRequest[] = [ { destination: { @@ -596,12 +596,12 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - Enricher.enrichGeoLocation(ctx); + await Enricher.enrichGeoLocation(ctx, () => {}); expect(ctx.request.body[0].message.traits).not.toHaveProperty('address'); expect(ctx.request.body[1].message.traits).not.toHaveProperty('address'); }); - test('should enrich when context.geo is populated correctly for multiple payloads with their own geolocation data', () => { + test('should enrich when context.geo is populated correctly for multiple payloads with their own geolocation data', async () => { const inputData: ProcessorTransformationRequest[] = [ { destination: { @@ -782,7 +782,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - Enricher.enrichGeoLocation(ctx); + await Enricher.enrichGeoLocation(ctx, () => {}); expect(ctx.request.body[0].message.traits).toMatchObject( expect.objectContaining({ age: 23, @@ -811,7 +811,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { }); describe('[GeoLocation Enrichment] Router/Batch transformation tests', () => { - test('should enrich with geo information when context.geo is present', () => { + test('should enrich with geo information when context.geo is present', async () => { const inputData: RouterTransformationRequest = { input: [ { @@ -998,7 +998,7 @@ describe('[GeoLocation Enrichment] Router/Batch transformation tests', () => { }; const ctx = { request: { body: inputData } }; // @ts-ignore - Enricher.enrichGeoLocation(ctx); + await Enricher.enrichGeoLocation(ctx, () => {}); expect(ctx.request.body.input[0].message.traits).toMatchObject( expect.objectContaining({ age: 23, From 5bdc2c6ba4094a4edc0f521e56d851bb0ec7783b Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Thu, 9 Nov 2023 17:34:38 +0530 Subject: [PATCH 11/21] fix: addressing issues - simplify logic for finding addressKey - update name of first matching key-value pair finder function - simplify logic for finding first key-value pair - add validation on message to be an object to perform search operation Signed-off-by: Sai Sankeerth --- src/helpers/geoLocation.ts | 15 +-- src/v0/util/index.js | 39 +++--- .../utils/getFirstMatchingKeyAndValue.test.ts | 127 ++++++++++++++++++ 3 files changed, 155 insertions(+), 26 deletions(-) create mode 100644 test/utils/getFirstMatchingKeyAndValue.test.ts diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts index 710c02c5ef..94b4420be5 100644 --- a/src/helpers/geoLocation.ts +++ b/src/helpers/geoLocation.ts @@ -2,7 +2,7 @@ import cloneDeep from 'lodash/cloneDeep'; import isEmpty from 'lodash/isEmpty'; import set from 'set-value'; import { FixMe } from '../util/types'; -import { getKeyAndValueFromMessage } from '../v0/util'; +import { getFirstMatchingKeyAndValue } from '../v0/util'; import * as GenericFieldMappingJson from '../v0/util/data/GenericFieldMapping.json'; export default class GeoLocationHelper { @@ -11,15 +11,10 @@ export default class GeoLocationHelper { address: Record; } { const { value: address, key: foundKey } = - getKeyAndValueFromMessage(message, GenericFieldMappingJson.address) || {}; - const { key: traitsKey } = getKeyAndValueFromMessage(message, GenericFieldMappingJson.traits); - let addressKey = foundKey; - if (!foundKey) { - addressKey = `${traitsKey}.address`; - if (!traitsKey) { - [addressKey] = GenericFieldMappingJson.address; - } - } + getFirstMatchingKeyAndValue(message, GenericFieldMappingJson.address) || {}; + const { key: traitsKey } = getFirstMatchingKeyAndValue(message, GenericFieldMappingJson.traits); + const addressKey = + foundKey || (traitsKey ? `${traitsKey}.address` : GenericFieldMappingJson.address[0]); return { addressKey, address }; } diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 15412d0b5f..5d1ea6adeb 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -659,23 +659,30 @@ const getValueFromMessage = (message, sourceKeys) => { return null; }; -const getKeyAndValueFromMessage = (message, sourceKeys) => { - if (Array.isArray(sourceKeys)) { - // got the possible sourceKeys - // eslint-disable-next-line no-restricted-syntax - for (const sourceKey of sourceKeys) { - const val = get(message, sourceKey); - if (val || val === false || val === 0) { - // return only if the value is valid. - // else look for next possible source in precedence - return { value: val, key: sourceKey }; - } +const getFirstMatchingKeyAndValue = (message, sourceKeys) => { + let srcKeys = sourceKeys; + if ( + typeof message !== 'object' || + (!Array.isArray(sourceKeys) && typeof sourceKeys !== 'string') + ) { + // sourceKeys => boolean, number, object, function etc,. + // message => anything other than object,. + return { value: null, key: '' }; + } + if (typeof sourceKeys === 'string') { + srcKeys = [sourceKeys]; + } + // got the possible sourceKeys + // eslint-disable-next-line no-restricted-syntax + for (const sourceKey of srcKeys) { + const val = get(message, sourceKey); + if (val || val === false || val === 0) { + // return only if the value is valid. + // else look for next possible source in precedence + return { value: val, key: sourceKey }; } - } else if (typeof sourceKeys === 'string') { - // got a single key - // - we don't need to iterate over a loop for a single possible value - return { value: get(message, sourceKeys), key: sourceKeys }; } + return { value: null, key: '' }; }; @@ -2212,5 +2219,5 @@ module.exports = { isValidInteger, isNewStatusCodesAccepted, IsGzipSupported, - getKeyAndValueFromMessage, + getFirstMatchingKeyAndValue, }; diff --git a/test/utils/getFirstMatchingKeyAndValue.test.ts b/test/utils/getFirstMatchingKeyAndValue.test.ts new file mode 100644 index 0000000000..84191066cd --- /dev/null +++ b/test/utils/getFirstMatchingKeyAndValue.test.ts @@ -0,0 +1,127 @@ +import util from '../../src/v0/util'; + +const getFirstMatchKVCases = [ + { + description: 'show value and key if present & sourceKeys are Array', + input: { + message: { + player: 'Roger Federer', + sport: 'Tennis', + rivals: { + spain: 'Rafael Nadal', + serbia: 'Novak Djokovic', + switzerland: 'Stan Wawrinka', + }, + }, + sourceKeys: ['spain.rivals', 'rivals.spain'], + }, + expectedOutput: { + value: 'Rafael Nadal', + key: 'rivals.spain', + }, + }, + { + description: + 'send value as null & key as empty string("") when the intended key is not present', + input: { + message: { + player: 'Roger Federer', + sport: 'Tennis', + rivals: ['Rafael Nadal', 'Novak Djokovic', 'Stan Wawrinka'], + }, + sourceKeys: ['spain.rivals', 'rivals.spain'], + }, + expectedOutput: { + value: null, + key: '', + }, + }, + { + description: + 'send correct value & key when the key is present & sourceKeys is of string data-type', + input: { + message: { + player: 'Roger Federer', + sport: 'Tennis', + rivals: ['Rafael Nadal', 'Novak Djokovic', 'Stan Wawrinka'], + }, + sourceKeys: 'rivals.1', + }, + expectedOutput: { + value: 'Novak Djokovic', + key: 'rivals.1', + }, + }, + { + description: 'send message null, sourceKeys as a valid string', + input: { + message: null, + sourceKeys: 'rivals.1', + }, + expectedOutput: { + value: null, + key: '', + }, + }, + { + description: 'send message undefined, sourceKeys as a valid string', + input: { + sourceKeys: 'rivals.1', + }, + expectedOutput: { + value: null, + key: '', + }, + }, + { + description: 'send message as empty string, sourceKeys as a valid string', + input: { + message: '', + sourceKeys: 'rivals.1', + }, + expectedOutput: { + value: null, + key: '', + }, + }, + { + description: 'send message as valid string(stringified json), sourceKeys as a valid string', + input: { + message: '{"rivals": ["a","b"]}', + sourceKeys: 'rivals.1', + }, + expectedOutput: { + value: null, + key: '', + }, + }, + { + description: 'send message as valid Array, sourceKeys as a valid string', + input: { + message: [1, 'va'], + sourceKeys: 'rivals.1', + }, + expectedOutput: { + value: null, + key: '', + }, + }, + { + description: 'send message as valid Array, sourceKeys as a valid string', + input: { + message: [{ a: { b: 2 } }, { c: { d: 4 } }], + sourceKeys: ['0.c.d', '1.c.d'], + }, + expectedOutput: { + value: 4, + key: '1.c.d', + }, + }, +]; + +describe('getFirstMatchingKeyAndValue tests', () => { + test.each(getFirstMatchKVCases)('$description', ({ input, expectedOutput }) => { + const actualOutput = util.getFirstMatchingKeyAndValue(input.message, input.sourceKeys); + expect(actualOutput).toEqual(expectedOutput); + }); +}); From a8259268cf649aa3dc5018c8454916334c1210e6 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Thu, 9 Nov 2023 17:46:07 +0530 Subject: [PATCH 12/21] fix: name of the returned object & name of the function Signed-off-by: Sai Sankeerth --- src/helpers/geoLocation.ts | 18 +++++++++++------- src/middlewares/enricher.ts | 2 +- test/helper/helper.test.ts | 22 +++++++++++----------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts index 94b4420be5..eb010ea3b9 100644 --- a/src/helpers/geoLocation.ts +++ b/src/helpers/geoLocation.ts @@ -7,24 +7,28 @@ import * as GenericFieldMappingJson from '../v0/util/data/GenericFieldMapping.js export default class GeoLocationHelper { public static getAddressKeyAndValue(message: Record): { - addressKey: string; - address: Record; + key: string; + value: Record; } { - const { value: address, key: foundKey } = - getFirstMatchingKeyAndValue(message, GenericFieldMappingJson.address) || {}; + const { value, key: foundKey } = getFirstMatchingKeyAndValue( + message, + GenericFieldMappingJson.address, + ); const { key: traitsKey } = getFirstMatchingKeyAndValue(message, GenericFieldMappingJson.traits); const addressKey = foundKey || (traitsKey ? `${traitsKey}.address` : GenericFieldMappingJson.address[0]); - return { addressKey, address }; + return { key: addressKey, value }; } - public static getGeoLocationData(message: Record): Record { + public static getMessageWithGeoLocationData( + message: Record, + ): Record { const msg = cloneDeep(message || {}); if (isEmpty(msg?.context?.geo || {})) { // geo-location data was not sent return {}; } - const { address, addressKey } = GeoLocationHelper.getAddressKeyAndValue(message); + const { value: address, key: addressKey } = GeoLocationHelper.getAddressKeyAndValue(message); const addressFieldMapping = { city: 'city', country: 'country', diff --git a/src/middlewares/enricher.ts b/src/middlewares/enricher.ts index 0b84b19ab0..53f05dcd68 100644 --- a/src/middlewares/enricher.ts +++ b/src/middlewares/enricher.ts @@ -13,7 +13,7 @@ export default class Enricher { data: RouterTransformationRequestData[], ): RouterTransformationRequestData[] { return data.map((inpEv) => { - const geoEnrichedMessage = GeoLocationHelper.getGeoLocationData(inpEv.message); + const geoEnrichedMessage = GeoLocationHelper.getMessageWithGeoLocationData(inpEv.message); return { ...inpEv, message: { diff --git a/test/helper/helper.test.ts b/test/helper/helper.test.ts index 1677f1e5b7..16c80bdfa7 100644 --- a/test/helper/helper.test.ts +++ b/test/helper/helper.test.ts @@ -59,7 +59,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); expect(enhancedMsg.context.traits.address).not.toBe(undefined); expect(enhancedMsg.context.traits.address).toEqual({ @@ -126,7 +126,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); expect(enhancedMsg.context.traits.address).not.toBe(undefined); expect(enhancedMsg.context.traits.address).toEqual({ @@ -199,7 +199,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); expect(enhancedMsg.traits.address).not.toBe(undefined); expect(enhancedMsg.traits.address).toEqual({ @@ -276,7 +276,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); expect(enhancedMsg.traits.address).not.toBe(undefined); expect(enhancedMsg.traits.address).toEqual({ @@ -351,7 +351,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); expect(enhancedMsg.traits.address).not.toBe(undefined); expect(enhancedMsg.traits.address).toEqual({ @@ -408,7 +408,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); expect(enhancedMsg).toEqual({}); }); @@ -416,7 +416,7 @@ describe('GeoLocationHelper tests', () => { describe('get addressKey & address tests', () => { test('addressKey should be "traits.address", when address is not present but traits object is present', () => { - const { addressKey } = GeoLocationHelper.getAddressKeyAndValue({ + const { key: addressKey } = GeoLocationHelper.getAddressKeyAndValue({ traits: { name: 'Bruce Wayne', age: 35, @@ -427,7 +427,7 @@ describe('get addressKey & address tests', () => { }); test('addressKey should be "context.traits.address", when address is not present but traits object is present', () => { - const { addressKey } = GeoLocationHelper.getAddressKeyAndValue({ + const { key: addressKey } = GeoLocationHelper.getAddressKeyAndValue({ context: { traits: { name: 'Bruce Wayne', @@ -440,7 +440,7 @@ describe('get addressKey & address tests', () => { }); test('addressKey should be "traits.address", when traits object is not present at all', () => { - const { addressKey } = GeoLocationHelper.getAddressKeyAndValue({ + const { key: addressKey } = GeoLocationHelper.getAddressKeyAndValue({ anonymousId: '129893-2idi9292', originalTimestamp: '2020-04-17T14:42:44.722Z', receivedAt: '2020-04-17T20:12:44.758+05:30', @@ -465,7 +465,7 @@ describe('get addressKey & address tests', () => { }); test('addressKey should be "context.traits.address", when address is present in context.traits & address is not present in traits', () => { - const { addressKey } = GeoLocationHelper.getAddressKeyAndValue({ + const { key: addressKey } = GeoLocationHelper.getAddressKeyAndValue({ anonymousId: '129893-2idi9292', originalTimestamp: '2020-04-17T14:42:44.722Z', receivedAt: '2020-04-17T20:12:44.758+05:30', @@ -499,7 +499,7 @@ describe('get addressKey & address tests', () => { }); test('addressKey should be "traits.address", when empty payload is sent', () => { - const { addressKey } = GeoLocationHelper.getAddressKeyAndValue({}); + const { key: addressKey } = GeoLocationHelper.getAddressKeyAndValue({}); expect(addressKey).toEqual('traits.address'); }); }); From b782b906721d063ec2918a023b28fd54fa3c0de8 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Thu, 9 Nov 2023 18:24:56 +0530 Subject: [PATCH 13/21] fix: addressing minor issues related to naming, used lodash.isNil instead of identifying the valid values manually, added test-cases Signed-off-by: Sai Sankeerth --- src/helpers/geoLocation.ts | 10 ++--- src/v0/util/index.js | 2 +- .../utils/getFirstMatchingKeyAndValue.test.ts | 38 +++++++++++++++++++ 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts index eb010ea3b9..b1f1c8b455 100644 --- a/src/helpers/geoLocation.ts +++ b/src/helpers/geoLocation.ts @@ -29,17 +29,17 @@ export default class GeoLocationHelper { return {}; } const { value: address, key: addressKey } = GeoLocationHelper.getAddressKeyAndValue(message); - const addressFieldMapping = { + const addressFieldToGeoFieldMap = { city: 'city', country: 'country', postalCode: 'postal', state: 'region', }; - const mappedAddress = Object.entries(addressFieldMapping).reduce( - (agg, [identifyAddressKey, geoKey]) => { - if (!address?.[identifyAddressKey] && msg?.context?.geo?.[geoKey]) { - return { [identifyAddressKey]: msg.context.geo[geoKey], ...agg }; + const mappedAddress = Object.entries(addressFieldToGeoFieldMap).reduce( + (agg, [addressFieldKey, geoFieldKey]) => { + if (!address?.[addressFieldKey] && msg?.context?.geo?.[geoFieldKey]) { + return { [addressFieldKey]: msg.context.geo[geoFieldKey], ...agg }; } return agg; }, diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 5d1ea6adeb..da7a3ded14 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -676,7 +676,7 @@ const getFirstMatchingKeyAndValue = (message, sourceKeys) => { // eslint-disable-next-line no-restricted-syntax for (const sourceKey of srcKeys) { const val = get(message, sourceKey); - if (val || val === false || val === 0) { + if (!lodash.isNil(val)) { // return only if the value is valid. // else look for next possible source in precedence return { value: val, key: sourceKey }; diff --git a/test/utils/getFirstMatchingKeyAndValue.test.ts b/test/utils/getFirstMatchingKeyAndValue.test.ts index 84191066cd..8481dd1ba2 100644 --- a/test/utils/getFirstMatchingKeyAndValue.test.ts +++ b/test/utils/getFirstMatchingKeyAndValue.test.ts @@ -117,6 +117,44 @@ const getFirstMatchKVCases = [ key: '1.c.d', }, }, + { + description: 'should get value as "0"', + input: { + message: { + player: 'Roger Federer', + sport: 'Tennis', + rivals: { + spain: 0, + serbia: 'Novak Djokovic', + switzerland: 'Stan Wawrinka', + }, + }, + sourceKeys: ['spain.rivals', 'rivals.spain'], + }, + expectedOutput: { + value: 0, + key: 'rivals.spain', + }, + }, + { + description: 'show get value as false', + input: { + message: { + player: 'Roger Federer', + sport: 'Tennis', + rivals: { + spain: false, + serbia: 'Novak Djokovic', + switzerland: 'Stan Wawrinka', + }, + }, + sourceKeys: ['spain.rivals', 'rivals.spain'], + }, + expectedOutput: { + value: false, + key: 'rivals.spain', + }, + }, ]; describe('getFirstMatchingKeyAndValue tests', () => { From 687f2fa83f3fb0cc6065a55a95328de43dd72b0d Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Thu, 9 Nov 2023 20:07:33 +0530 Subject: [PATCH 14/21] fix: correcting test-case Signed-off-by: Sai Sankeerth --- test/helper/helper.test.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/helper/helper.test.ts b/test/helper/helper.test.ts index 16c80bdfa7..52ba215acb 100644 --- a/test/helper/helper.test.ts +++ b/test/helper/helper.test.ts @@ -101,10 +101,6 @@ describe('GeoLocationHelper tests', () => { screen: { density: 2, }, - traits: { - email: 'example124@email.com', - name: 'abcd124', - }, userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', }; @@ -128,8 +124,8 @@ describe('GeoLocationHelper tests', () => { const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); - expect(enhancedMsg.context.traits.address).not.toBe(undefined); - expect(enhancedMsg.context.traits.address).toEqual({ + expect(enhancedMsg.traits.address).not.toBe(undefined); + expect(enhancedMsg.traits.address).toEqual({ city: 'Gurugram', country: 'IN', postalCode: '122001', From e4c6bb27591672ba369750c6a1c03bfa600fe4d7 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Thu, 9 Nov 2023 20:08:48 +0530 Subject: [PATCH 15/21] fix: cloneDeep for all, add test-cases to reflect same referencing & null message cases Signed-off-by: Sai Sankeerth --- src/helpers/geoLocation.ts | 5 +- test/helper/helper.test.ts | 17 ++ .../destinations/webhook/processor/data.ts | 249 ++++++++++++++++++ 3 files changed, 269 insertions(+), 2 deletions(-) diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts index b1f1c8b455..fc80d74e6c 100644 --- a/src/helpers/geoLocation.ts +++ b/src/helpers/geoLocation.ts @@ -23,8 +23,8 @@ export default class GeoLocationHelper { public static getMessageWithGeoLocationData( message: Record, ): Record { - const msg = cloneDeep(message || {}); - if (isEmpty(msg?.context?.geo || {})) { + let msg = message; + if (isEmpty(msg?.context?.geo)) { // geo-location data was not sent return {}; } @@ -46,6 +46,7 @@ export default class GeoLocationHelper { {}, ); if (!isEmpty(address) || !isEmpty(mappedAddress)) { + msg = cloneDeep(message || {}); set(msg, addressKey, { ...address, ...mappedAddress }); } return msg; diff --git a/test/helper/helper.test.ts b/test/helper/helper.test.ts index 52ba215acb..5eac756736 100644 --- a/test/helper/helper.test.ts +++ b/test/helper/helper.test.ts @@ -283,6 +283,8 @@ describe('GeoLocationHelper tests', () => { city: 'Dharamshala', postalCode: '123546', }); + // same reference check + expect(enhancedMsg.traits.address).toStrictEqual(msg.traits.address); }); test("when context.geo doesn't have some properties, do not make use of non-available values in context.geo", () => { @@ -407,6 +409,21 @@ describe('GeoLocationHelper tests', () => { const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); expect(enhancedMsg).toEqual({}); + // @ts-ignore + expect(enhancedMsg?.traits?.address).toBe(undefined); + // @ts-ignore + expect(enhancedMsg?.context?.traits?.address).toBe(undefined); + }); + + test('when message is null(no geo-enrichment happened), enrichment would not happen', () => { + const msg = null; + + // @ts-ignore + const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); + + expect(enhancedMsg).toEqual({}); + // @ts-ignore + expect(enhancedMsg?.traits).toBe(undefined); }); }); diff --git a/test/integrations/destinations/webhook/processor/data.ts b/test/integrations/destinations/webhook/processor/data.ts index d02fa58227..92a67170e0 100644 --- a/test/integrations/destinations/webhook/processor/data.ts +++ b/test/integrations/destinations/webhook/processor/data.ts @@ -1,3 +1,251 @@ +const geoLocationData = [ + { + name: 'webhook', + description: '[GeoLocation] Test 0', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + context: { + device: { + id: 'df16bffa-5c3d-4fbb-9bce-3bab098129a7R', + manufacturer: 'Xiaomi', + model: 'Redmi 6', + name: 'xiaomi', + }, + network: { carrier: 'Banglalink' }, + os: { name: 'android', version: '8.1.0' }, + traits: { + address: { city: 'Dhaka', country: 'Bangladesh' }, + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + }, + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + }, + event: 'spin_result', + integrations: { All: true }, + message_id: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + properties: { + additional_bet_index: 0, + battle_id: 'N/A', + bet_amount: 9, + bet_level: 1, + bet_multiplier: 1, + coin_balance: 9466052, + current_module_name: 'CasinoGameModule', + days_in_game: 0, + extra_param: 'N/A', + fb_profile: '0', + featureGameType: 'N/A', + game_fps: 30, + game_id: 'fireEagleBase', + game_name: 'FireEagleSlots', + gem_balance: 0, + graphicsQuality: 'HD', + idfa: '2bf99787-33d2-4ae2-a76a-c49672f97252', + internetReachability: 'ReachableViaLocalAreaNetwork', + isLowEndDevice: 'False', + is_auto_spin: 'False', + is_turbo: 'False', + isf: 'False', + ishighroller: 'False', + jackpot_win_amount: 90, + jackpot_win_type: 'Silver', + level: 6, + lifetime_gem_balance: 0, + no_of_spin: 1, + player_total_battles: 0, + player_total_shields: 0, + start_date: '2019-08-01', + total_payments: 0, + tournament_id: 'T1561970819', + userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + versionSessionCount: 2, + win_amount: 0, + }, + timestamp: '2019-09-01T15:46:51.693229+05:30', + type: 'track', + user_properties: { + coin_balance: 9466052, + current_module_name: 'CasinoGameModule', + fb_profile: '0', + game_fps: 30, + game_name: 'FireEagleSlots', + gem_balance: 0, + graphicsQuality: 'HD', + idfa: '2bf99787-33d2-4ae2-a76a-c49672f97252', + internetReachability: 'ReachableViaLocalAreaNetwork', + isLowEndDevice: false, + level: 6, + lifetime_gem_balance: 0, + player_total_battles: 0, + player_total_shields: 0, + start_date: '2019-08-01', + total_payments: 0, + userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + versionSessionCount: 2, + }, + }, + destination: { + Config: { + webhookUrl: 'http://6b0e6a60.ngrok.io', + headers: [ + { from: '', to: '' }, + { from: 'test2', to: 'value2' }, + ], + }, + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + }, + metadata: { + destinationId: 'd1', + workspaceId: 'w1', + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + destinationId: 'd1', + workspaceId: 'w1', + }, + output: { + body: { + XML: {}, + JSON_ARRAY: {}, + JSON: { + timestamp: '2019-09-01T15:46:51.693229+05:30', + user_properties: { + total_payments: 0, + internetReachability: 'ReachableViaLocalAreaNetwork', + level: 6, + userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + coin_balance: 9466052, + player_total_shields: 0, + isLowEndDevice: false, + game_fps: 30, + idfa: '2bf99787-33d2-4ae2-a76a-c49672f97252', + graphicsQuality: 'HD', + current_module_name: 'CasinoGameModule', + player_total_battles: 0, + lifetime_gem_balance: 0, + gem_balance: 0, + fb_profile: '0', + start_date: '2019-08-01', + versionSessionCount: 2, + game_name: 'FireEagleSlots', + }, + integrations: { All: true }, + event: 'spin_result', + message_id: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + context: { + geo: { + city: 'Gurugram', + country: 'IN', + ip: '223.190.82.63', + location: '28.459700,77.028200', + postal: '122001', + region: 'Haryana', + timezone: 'Asia/Kolkata', + }, + device: { + model: 'Redmi 6', + manufacturer: 'Xiaomi', + id: 'df16bffa-5c3d-4fbb-9bce-3bab098129a7R', + name: 'xiaomi', + }, + traits: { + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + address: { + city: 'Dhaka', + country: 'Bangladesh', + postalCode: '122001', + state: 'Haryana', + }, + }, + os: { version: '8.1.0', name: 'android' }, + network: { carrier: 'Banglalink' }, + }, + type: 'track', + properties: { + userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + jackpot_win_type: 'Silver', + coin_balance: 9466052, + bet_level: 1, + ishighroller: 'False', + tournament_id: 'T1561970819', + battle_id: 'N/A', + bet_amount: 9, + fb_profile: '0', + player_total_shields: 0, + is_turbo: 'False', + player_total_battles: 0, + bet_multiplier: 1, + start_date: '2019-08-01', + versionSessionCount: 2, + graphicsQuality: 'HD', + is_auto_spin: 'False', + days_in_game: 0, + additional_bet_index: 0, + isLowEndDevice: 'False', + game_fps: 30, + extra_param: 'N/A', + idfa: '2bf99787-33d2-4ae2-a76a-c49672f97252', + current_module_name: 'CasinoGameModule', + game_id: 'fireEagleBase', + featureGameType: 'N/A', + gem_balance: 0, + internetReachability: 'ReachableViaLocalAreaNetwork', + total_payments: 0, + level: 6, + win_amount: 0, + no_of_spin: 1, + game_name: 'FireEagleSlots', + jackpot_win_amount: 90, + lifetime_gem_balance: 0, + isf: 'False', + }, + }, + FORM: {}, + }, + files: {}, + endpoint: 'http://6b0e6a60.ngrok.io', + userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + headers: { 'content-type': 'application/json', test2: 'value2' }, + version: '1', + params: {}, + type: 'REST', + method: 'POST', + }, + statusCode: 200, + }, + ], + }, + }, + }, +]; export const data = [ { name: 'webhook', @@ -2167,4 +2415,5 @@ export const data = [ }, }, }, + ...geoLocationData, ]; From 2c449996962f5c6488e20f567bf6f755df816f72 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Thu, 9 Nov 2023 22:20:57 +0530 Subject: [PATCH 16/21] fix: address comments in test-cases Signed-off-by: Sai Sankeerth --- .../destinations/webhook/processor/data.ts | 102 +----------------- test/middleware/enricher.test.ts | 7 +- .../utils/getFirstMatchingKeyAndValue.test.ts | 24 +++++ 3 files changed, 30 insertions(+), 103 deletions(-) diff --git a/test/integrations/destinations/webhook/processor/data.ts b/test/integrations/destinations/webhook/processor/data.ts index 92a67170e0..f9f2643a4c 100644 --- a/test/integrations/destinations/webhook/processor/data.ts +++ b/test/integrations/destinations/webhook/processor/data.ts @@ -40,62 +40,12 @@ const geoLocationData = [ properties: { additional_bet_index: 0, battle_id: 'N/A', - bet_amount: 9, - bet_level: 1, - bet_multiplier: 1, - coin_balance: 9466052, - current_module_name: 'CasinoGameModule', - days_in_game: 0, - extra_param: 'N/A', - fb_profile: '0', - featureGameType: 'N/A', - game_fps: 30, - game_id: 'fireEagleBase', - game_name: 'FireEagleSlots', - gem_balance: 0, - graphicsQuality: 'HD', - idfa: '2bf99787-33d2-4ae2-a76a-c49672f97252', - internetReachability: 'ReachableViaLocalAreaNetwork', - isLowEndDevice: 'False', - is_auto_spin: 'False', - is_turbo: 'False', - isf: 'False', - ishighroller: 'False', - jackpot_win_amount: 90, - jackpot_win_type: 'Silver', - level: 6, - lifetime_gem_balance: 0, - no_of_spin: 1, - player_total_battles: 0, - player_total_shields: 0, - start_date: '2019-08-01', - total_payments: 0, - tournament_id: 'T1561970819', - userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', - versionSessionCount: 2, - win_amount: 0, }, timestamp: '2019-09-01T15:46:51.693229+05:30', type: 'track', user_properties: { coin_balance: 9466052, current_module_name: 'CasinoGameModule', - fb_profile: '0', - game_fps: 30, - game_name: 'FireEagleSlots', - gem_balance: 0, - graphicsQuality: 'HD', - idfa: '2bf99787-33d2-4ae2-a76a-c49672f97252', - internetReachability: 'ReachableViaLocalAreaNetwork', - isLowEndDevice: false, - level: 6, - lifetime_gem_balance: 0, - player_total_battles: 0, - player_total_shields: 0, - start_date: '2019-08-01', - total_payments: 0, - userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', - versionSessionCount: 2, }, }, destination: { @@ -137,24 +87,8 @@ const geoLocationData = [ JSON: { timestamp: '2019-09-01T15:46:51.693229+05:30', user_properties: { - total_payments: 0, - internetReachability: 'ReachableViaLocalAreaNetwork', - level: 6, - userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', coin_balance: 9466052, - player_total_shields: 0, - isLowEndDevice: false, - game_fps: 30, - idfa: '2bf99787-33d2-4ae2-a76a-c49672f97252', - graphicsQuality: 'HD', current_module_name: 'CasinoGameModule', - player_total_battles: 0, - lifetime_gem_balance: 0, - gem_balance: 0, - fb_profile: '0', - start_date: '2019-08-01', - versionSessionCount: 2, - game_name: 'FireEagleSlots', }, integrations: { All: true }, event: 'spin_result', @@ -190,42 +124,8 @@ const geoLocationData = [ }, type: 'track', properties: { - userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', - jackpot_win_type: 'Silver', - coin_balance: 9466052, - bet_level: 1, - ishighroller: 'False', - tournament_id: 'T1561970819', - battle_id: 'N/A', - bet_amount: 9, - fb_profile: '0', - player_total_shields: 0, - is_turbo: 'False', - player_total_battles: 0, - bet_multiplier: 1, - start_date: '2019-08-01', - versionSessionCount: 2, - graphicsQuality: 'HD', - is_auto_spin: 'False', - days_in_game: 0, additional_bet_index: 0, - isLowEndDevice: 'False', - game_fps: 30, - extra_param: 'N/A', - idfa: '2bf99787-33d2-4ae2-a76a-c49672f97252', - current_module_name: 'CasinoGameModule', - game_id: 'fireEagleBase', - featureGameType: 'N/A', - gem_balance: 0, - internetReachability: 'ReachableViaLocalAreaNetwork', - total_payments: 0, - level: 6, - win_amount: 0, - no_of_spin: 1, - game_name: 'FireEagleSlots', - jackpot_win_amount: 90, - lifetime_gem_balance: 0, - isf: 'False', + battle_id: 'N/A', }, }, FORM: {}, diff --git a/test/middleware/enricher.test.ts b/test/middleware/enricher.test.ts index 23fb3ee924..e058e11685 100644 --- a/test/middleware/enricher.test.ts +++ b/test/middleware/enricher.test.ts @@ -345,7 +345,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ); }); - test('should not enrich when context.geo is not populated', async () => { + test('should not enrich when context.geo is not populated & address is already present', async () => { const inputData: ProcessorTransformationRequest[] = [ { destination: { @@ -420,6 +420,9 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { age: 23, email: 'testmp@rudderstack.com', firstname: 'Test Kafka', + address: { + country: 'India', + }, }, timestamp: '2020-04-17T20:12:44.758+05:30', type: 'identify', @@ -432,7 +435,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { await Enricher.enrichGeoLocation(ctx, () => {}); // @ts-ignore - expect(ctx.request.body[0].message.traits?.address).toBe(undefined); + expect(ctx.request.body[0].message.traits?.address).toEqual({ country: 'India' }); }); test('should not contain address object, when context.geo & address are not present', async () => { diff --git a/test/utils/getFirstMatchingKeyAndValue.test.ts b/test/utils/getFirstMatchingKeyAndValue.test.ts index 8481dd1ba2..08a3149259 100644 --- a/test/utils/getFirstMatchingKeyAndValue.test.ts +++ b/test/utils/getFirstMatchingKeyAndValue.test.ts @@ -155,6 +155,30 @@ const getFirstMatchKVCases = [ key: 'rivals.spain', }, }, + { + description: 'should get first available value(in this case "traits.address")', + input: { + message: { + context: { + traits: { + address: { + country: 'Switzerland', + }, + }, + }, + traits: { + address: { + country: 'Bulgaria', + }, + }, + }, + sourceKeys: ['traits.address', 'context.traits.address'], + }, + expectedOutput: { + value: { country: 'Bulgaria' }, + key: 'traits.address', + }, + }, ]; describe('getFirstMatchingKeyAndValue tests', () => { From 8928020e9a614832d834fbf98437a2192247adbd Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Fri, 10 Nov 2023 10:38:07 +0530 Subject: [PATCH 17/21] fix: unnecessary test-case data for enricher unit-tests Signed-off-by: Sai Sankeerth --- test/middleware/enricher.test.ts | 262 ------------------------------- 1 file changed, 262 deletions(-) diff --git a/test/middleware/enricher.test.ts b/test/middleware/enricher.test.ts index e058e11685..52b74cd9ba 100644 --- a/test/middleware/enricher.test.ts +++ b/test/middleware/enricher.test.ts @@ -143,8 +143,6 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { IsProcessorEnabled: true, }, message: { - anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', - channel: 'web', context: { geo: { city: 'Gurugram', @@ -155,42 +153,14 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { region: 'Haryana', timezone: 'Asia/Kolkata', }, - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.1.1-rc.2', - }, library: { name: 'RudderLabs JavaScript SDK', version: '1.1.1-rc.2', }, - locale: 'en-GB', - os: { - name: '', - version: '', - }, - page: { - path: '/tests/html/index4.html', - referrer: '', - search: '', - title: '', - url: 'http://localhost/tests/html/index4.html', - }, - screen: { - density: 2, - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', }, integrations: { All: true, }, - messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', - originalTimestamp: '2020-04-17T14:42:44.722Z', - receivedAt: '2020-04-17T20:12:44.758+05:30', - request_ip: '[::1]:53513', - sentAt: '2020-04-17T14:42:44.722Z', traits: { age: 23, email: 'testmp1@rudderstack.com', @@ -203,9 +173,6 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { city: 'Bandarpur', }, }, - timestamp: '2020-04-17T20:12:44.758+05:30', - type: 'identify', - userId: 'user12345', }, }, ]; @@ -260,8 +227,6 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { IsProcessorEnabled: true, }, message: { - anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', - channel: 'web', context: { geo: { city: 'Gurugram', @@ -272,42 +237,11 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { region: 'Haryana', timezone: 'Asia/Kolkata', }, - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.1.1-rc.2', - }, library: { name: 'RudderLabs JavaScript SDK', version: '1.1.1-rc.2', }, - locale: 'en-GB', - os: { - name: '', - version: '', - }, - page: { - path: '/tests/html/index4.html', - referrer: '', - search: '', - title: '', - url: 'http://localhost/tests/html/index4.html', - }, - screen: { - density: 2, - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', - }, - integrations: { - All: true, }, - messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', - originalTimestamp: '2020-04-17T14:42:44.722Z', - receivedAt: '2020-04-17T20:12:44.758+05:30', - request_ip: '[::1]:53513', - sentAt: '2020-04-17T14:42:44.722Z', traits: { age: 23, email: 'testmp1@rudderstack.com', @@ -319,9 +253,6 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { postalCode: '123321', }, }, - timestamp: '2020-04-17T20:12:44.758+05:30', - type: 'identify', - userId: 'user12345', }, }, ]; @@ -377,45 +308,12 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { IsProcessorEnabled: true, }, message: { - anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', - channel: 'web', context: { - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.1.1-rc.2', - }, library: { name: 'RudderLabs JavaScript SDK', version: '1.1.1-rc.2', }, - locale: 'en-GB', - os: { - name: '', - version: '', - }, - page: { - path: '/tests/html/index4.html', - referrer: '', - search: '', - title: '', - url: 'http://localhost/tests/html/index4.html', - }, - screen: { - density: 2, - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', - }, - integrations: { - All: true, }, - messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', - originalTimestamp: '2020-04-17T14:42:44.722Z', - receivedAt: '2020-04-17T20:12:44.758+05:30', - request_ip: '[::1]:53513', - sentAt: '2020-04-17T14:42:44.722Z', traits: { age: 23, email: 'testmp@rudderstack.com', @@ -424,9 +322,6 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { country: 'India', }, }, - timestamp: '2020-04-17T20:12:44.758+05:30', - type: 'identify', - userId: 'user12345', }, }, ]; @@ -549,51 +444,15 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { IsProcessorEnabled: true, }, message: { - anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', - channel: 'web', context: { - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.1.1-rc.2', - }, library: { name: 'RudderLabs JavaScript SDK', version: '1.1.1-rc.2', }, - locale: 'en-GB', - os: { - name: '', - version: '', - }, - page: { - path: '/tests/html/index4.html', - referrer: '', - search: '', - title: '', - url: 'http://localhost/tests/html/index4.html', - }, - screen: { - density: 2, - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', - }, - integrations: { - All: true, }, - messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', - originalTimestamp: '2020-04-17T14:42:44.722Z', - receivedAt: '2020-04-17T20:12:44.758+05:30', - request_ip: '[::1]:53513', - sentAt: '2020-04-17T14:42:44.722Z', traits: { key: 'val', }, - timestamp: '2020-04-17T20:12:44.758+05:30', - type: 'identify', - userId: 'user12345', }, }, ]; @@ -636,8 +495,6 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { IsProcessorEnabled: true, }, message: { - anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', - channel: 'web', context: { geo: { city: 'Gurugram', @@ -648,50 +505,16 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { region: 'Haryana', timezone: 'Asia/Kolkata', }, - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.1.1-rc.2', - }, library: { name: 'RudderLabs JavaScript SDK', version: '1.1.1-rc.2', }, - locale: 'en-GB', - os: { - name: '', - version: '', - }, - page: { - path: '/tests/html/index4.html', - referrer: '', - search: '', - title: '', - url: 'http://localhost/tests/html/index4.html', - }, - screen: { - density: 2, - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', }, - integrations: { - All: true, - }, - messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', - originalTimestamp: '2020-04-17T14:42:44.722Z', - receivedAt: '2020-04-17T20:12:44.758+05:30', - request_ip: '[::1]:53513', - sentAt: '2020-04-17T14:42:44.722Z', traits: { age: 23, email: 'testmp@rudderstack.com', firstname: 'Test Kafka', }, - timestamp: '2020-04-17T20:12:44.758+05:30', - type: 'identify', - userId: 'user12345', }, }, { @@ -846,8 +669,6 @@ describe('[GeoLocation Enrichment] Router/Batch transformation tests', () => { IsProcessorEnabled: true, }, message: { - anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', - channel: 'web', context: { geo: { city: 'Gurugram', @@ -858,50 +679,12 @@ describe('[GeoLocation Enrichment] Router/Batch transformation tests', () => { region: 'Haryana', timezone: 'Asia/Kolkata', }, - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.1.1-rc.2', - }, - library: { - name: 'RudderLabs JavaScript SDK', - version: '1.1.1-rc.2', - }, - locale: 'en-GB', - os: { - name: '', - version: '', - }, - page: { - path: '/tests/html/index4.html', - referrer: '', - search: '', - title: '', - url: 'http://localhost/tests/html/index4.html', - }, - screen: { - density: 2, - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', }, - integrations: { - All: true, - }, - messageId: 'fad9b3fb-5778-4db3-9fb6-7168b554191f', - originalTimestamp: '2020-04-17T14:42:44.722Z', - receivedAt: '2020-04-17T20:12:44.758+05:30', - request_ip: '[::1]:53513', - sentAt: '2020-04-17T14:42:44.722Z', traits: { age: 23, email: 'testm3p@rudderstack.com', firstname: 'Test Kafka', }, - timestamp: '2020-04-17T20:12:44.758+05:30', - type: 'identify', - userId: 'user12345', }, }, { @@ -933,8 +716,6 @@ describe('[GeoLocation Enrichment] Router/Batch transformation tests', () => { IsProcessorEnabled: true, }, message: { - anonymousId: 'ac7722c2-ccb6-4ae2-baf6-1effe861f4cd', - channel: 'web', context: { geo: { city: 'Gurugram', @@ -945,55 +726,12 @@ describe('[GeoLocation Enrichment] Router/Batch transformation tests', () => { region: 'Haryana', timezone: 'Asia/Kolkata', }, - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.1.1-rc.2', - }, - library: { - name: 'RudderLabs JavaScript SDK', - version: '1.1.1-rc.2', - }, - locale: 'en-GB', - os: { - name: '', - version: '', - }, - page: { - path: '/tests/html/index4.html', - referrer: '', - search: '', - title: '', - url: 'http://localhost/tests/html/index4.html', - }, - screen: { - density: 2, - }, traits: { age: 23, email: 'testmp@rudderstack.com', firstname: 'Test Kafka', }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36', - }, - event: 'test track with property', - integrations: { - All: true, - }, - messageId: '37b75e61-9bd2-4fb8-91ed-e3a064905f3a', - originalTimestamp: '2020-04-17T14:42:44.724Z', - properties: { - test_prop_1: 'test prop', - test_prop_2: 1232, }, - receivedAt: '2020-04-17T20:12:44.758+05:30', - request_ip: '[::1]:53512', - sentAt: '2020-04-17T14:42:44.725Z', - timestamp: '2020-04-17T20:12:44.757+05:30', - type: 'track', - userId: 'user12345', }, }, ], From 25fb0d73fe7eb491e09810b9ec6a36a247c2e119 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Fri, 10 Nov 2023 14:50:25 +0530 Subject: [PATCH 18/21] fix: addressing comments - 2 - update enricher class name, method, var name & update related test-cases - return value as undefined when anything apart from Array or object is sent for searching - make use of msg rather than message Signed-off-by: Sai Sankeerth --- src/helpers/geoLocation.ts | 4 ++-- src/middlewares/enricher.ts | 18 ++++++++++-------- src/routes/destination.ts | 8 ++++---- src/v0/util/index.js | 2 +- test/helper/helper.test.ts | 14 +++++++------- test/middleware/enricher.test.ts | 16 ++++++++-------- test/utils/getFirstMatchingKeyAndValue.test.ts | 3 --- 7 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts index fc80d74e6c..4becad206a 100644 --- a/src/helpers/geoLocation.ts +++ b/src/helpers/geoLocation.ts @@ -20,7 +20,7 @@ export default class GeoLocationHelper { return { key: addressKey, value }; } - public static getMessageWithGeoLocationData( + public static enrichMessageAddressWithGeoData( message: Record, ): Record { let msg = message; @@ -28,7 +28,7 @@ export default class GeoLocationHelper { // geo-location data was not sent return {}; } - const { value: address, key: addressKey } = GeoLocationHelper.getAddressKeyAndValue(message); + const { value: address, key: addressKey } = GeoLocationHelper.getAddressKeyAndValue(msg); const addressFieldToGeoFieldMap = { city: 'city', country: 'country', diff --git a/src/middlewares/enricher.ts b/src/middlewares/enricher.ts index 53f05dcd68..ba46f4d6ed 100644 --- a/src/middlewares/enricher.ts +++ b/src/middlewares/enricher.ts @@ -8,23 +8,25 @@ import GeoLocationHelper from '../helpers/geoLocation'; export type DTRequest = RouterTransformationRequest | ProcessorTransformationRequest[]; -export default class Enricher { +export default class GeoEnricher { private static enrichWithGeoInfo( data: RouterTransformationRequestData[], ): RouterTransformationRequestData[] { - return data.map((inpEv) => { - const geoEnrichedMessage = GeoLocationHelper.getMessageWithGeoLocationData(inpEv.message); + return data.map((inputEvent) => { + const geoEnrichedMessage = GeoLocationHelper.enrichMessageAddressWithGeoData( + inputEvent.message, + ); return { - ...inpEv, + ...inputEvent, message: { - ...inpEv.message, + ...inputEvent.message, ...geoEnrichedMessage, }, }; }); } - public static async enrichGeoLocation(ctx: Context, next: Next) { + public static async enrich(ctx: Context, next: Next) { const transformationRequest = ctx.request.body; let transformationReq: DTRequest; let reqBody: unknown; @@ -34,7 +36,7 @@ export default class Enricher { if (isRouterTransform) { // Router or batch transformation request transformationReq = transformationRequest as RouterTransformationRequest; - const enrichedEvents: RouterTransformationRequestData[] = Enricher.enrichWithGeoInfo( + const enrichedEvents: RouterTransformationRequestData[] = GeoEnricher.enrichWithGeoInfo( transformationReq.input, ); reqBody = { @@ -44,7 +46,7 @@ export default class Enricher { } else { // Processor transformation transformationReq = transformationRequest as ProcessorTransformationRequest[]; - reqBody = Enricher.enrichWithGeoInfo(transformationReq); + reqBody = GeoEnricher.enrichWithGeoInfo(transformationReq); } ctx.request.body = reqBody; await next(); diff --git a/src/routes/destination.ts b/src/routes/destination.ts index e8caaf99e7..96f79a6c3c 100644 --- a/src/routes/destination.ts +++ b/src/routes/destination.ts @@ -3,7 +3,7 @@ import DestinationController from '../controllers/destination'; import RegulationController from '../controllers/regulation'; import FeatureFlagController from '../middlewares/featureFlag'; import RouteActivationController from '../middlewares/routeActivation'; -import Enricher from '../middlewares/enricher'; +import GeoEnricher from '../middlewares/enricher'; const router = new Router(); @@ -12,7 +12,7 @@ router.post( RouteActivationController.isDestinationRouteActive, RouteActivationController.destinationProcFilter, FeatureFlagController.handle, - Enricher.enrichGeoLocation, + GeoEnricher.enrich, DestinationController.destinationTransformAtProcessor, ); router.post( @@ -20,7 +20,7 @@ router.post( RouteActivationController.isDestinationRouteActive, RouteActivationController.destinationRtFilter, FeatureFlagController.handle, - Enricher.enrichGeoLocation, + GeoEnricher.enrich, DestinationController.destinationTransformAtRouter, ); router.post( @@ -28,7 +28,7 @@ router.post( RouteActivationController.isDestinationRouteActive, RouteActivationController.destinationBatchFilter, FeatureFlagController.handle, - Enricher.enrichGeoLocation, + GeoEnricher.enrich, DestinationController.batchProcess, ); diff --git a/src/v0/util/index.js b/src/v0/util/index.js index da7a3ded14..70ec9055cc 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -667,7 +667,7 @@ const getFirstMatchingKeyAndValue = (message, sourceKeys) => { ) { // sourceKeys => boolean, number, object, function etc,. // message => anything other than object,. - return { value: null, key: '' }; + return { key: '' }; } if (typeof sourceKeys === 'string') { srcKeys = [sourceKeys]; diff --git a/test/helper/helper.test.ts b/test/helper/helper.test.ts index 5eac756736..ebb78ee2a2 100644 --- a/test/helper/helper.test.ts +++ b/test/helper/helper.test.ts @@ -59,7 +59,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.enrichMessageAddressWithGeoData(msg); expect(enhancedMsg.context.traits.address).not.toBe(undefined); expect(enhancedMsg.context.traits.address).toEqual({ @@ -122,7 +122,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.enrichMessageAddressWithGeoData(msg); expect(enhancedMsg.traits.address).not.toBe(undefined); expect(enhancedMsg.traits.address).toEqual({ @@ -195,7 +195,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.enrichMessageAddressWithGeoData(msg); expect(enhancedMsg.traits.address).not.toBe(undefined); expect(enhancedMsg.traits.address).toEqual({ @@ -272,7 +272,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.enrichMessageAddressWithGeoData(msg); expect(enhancedMsg.traits.address).not.toBe(undefined); expect(enhancedMsg.traits.address).toEqual({ @@ -349,7 +349,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.enrichMessageAddressWithGeoData(msg); expect(enhancedMsg.traits.address).not.toBe(undefined); expect(enhancedMsg.traits.address).toEqual({ @@ -406,7 +406,7 @@ describe('GeoLocationHelper tests', () => { userId: 'abcd-124', }; - const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.enrichMessageAddressWithGeoData(msg); expect(enhancedMsg).toEqual({}); // @ts-ignore @@ -419,7 +419,7 @@ describe('GeoLocationHelper tests', () => { const msg = null; // @ts-ignore - const enhancedMsg = GeoLocationHelper.getMessageWithGeoLocationData(msg); + const enhancedMsg = GeoLocationHelper.enrichMessageAddressWithGeoData(msg); expect(enhancedMsg).toEqual({}); // @ts-ignore diff --git a/test/middleware/enricher.test.ts b/test/middleware/enricher.test.ts index 52b74cd9ba..de029d90a2 100644 --- a/test/middleware/enricher.test.ts +++ b/test/middleware/enricher.test.ts @@ -1,5 +1,5 @@ import { ProcessorTransformationRequest, RouterTransformationRequest } from '../../src/types'; -import Enricher from '../../src/middlewares/enricher'; +import GeoEnricher from '../../src/middlewares/enricher'; describe('[GeoLocation Enrichment] Processor transformation tests', () => { test('should enrich when context.geo is populated correctly', async () => { @@ -95,7 +95,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - await Enricher.enrichGeoLocation(ctx, () => {}); + await GeoEnricher.enrich(ctx, () => {}); expect(ctx.request.body[0].message.traits).toMatchObject( expect.objectContaining({ age: 23, @@ -178,7 +178,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - await Enricher.enrichGeoLocation(ctx, () => {}); + await GeoEnricher.enrich(ctx, () => {}); expect(ctx.request.body[0].message.traits).toMatchObject( expect.objectContaining({ age: 23, @@ -258,7 +258,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - await Enricher.enrichGeoLocation(ctx, () => {}); + await GeoEnricher.enrich(ctx, () => {}); expect(ctx.request.body[0].message.traits).toMatchObject( expect.objectContaining({ age: 23, @@ -327,7 +327,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - await Enricher.enrichGeoLocation(ctx, () => {}); + await GeoEnricher.enrich(ctx, () => {}); // @ts-ignore expect(ctx.request.body[0].message.traits?.address).toEqual({ country: 'India' }); @@ -458,7 +458,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - await Enricher.enrichGeoLocation(ctx, () => {}); + await GeoEnricher.enrich(ctx, () => {}); expect(ctx.request.body[0].message.traits).not.toHaveProperty('address'); expect(ctx.request.body[1].message.traits).not.toHaveProperty('address'); }); @@ -608,7 +608,7 @@ describe('[GeoLocation Enrichment] Processor transformation tests', () => { ]; const ctx = { request: { body: inputData } }; // @ts-ignore - await Enricher.enrichGeoLocation(ctx, () => {}); + await GeoEnricher.enrich(ctx, () => {}); expect(ctx.request.body[0].message.traits).toMatchObject( expect.objectContaining({ age: 23, @@ -739,7 +739,7 @@ describe('[GeoLocation Enrichment] Router/Batch transformation tests', () => { }; const ctx = { request: { body: inputData } }; // @ts-ignore - await Enricher.enrichGeoLocation(ctx, () => {}); + await GeoEnricher.enrich(ctx, () => {}); expect(ctx.request.body.input[0].message.traits).toMatchObject( expect.objectContaining({ age: 23, diff --git a/test/utils/getFirstMatchingKeyAndValue.test.ts b/test/utils/getFirstMatchingKeyAndValue.test.ts index 08a3149259..fac7ae9670 100644 --- a/test/utils/getFirstMatchingKeyAndValue.test.ts +++ b/test/utils/getFirstMatchingKeyAndValue.test.ts @@ -69,7 +69,6 @@ const getFirstMatchKVCases = [ sourceKeys: 'rivals.1', }, expectedOutput: { - value: null, key: '', }, }, @@ -80,7 +79,6 @@ const getFirstMatchKVCases = [ sourceKeys: 'rivals.1', }, expectedOutput: { - value: null, key: '', }, }, @@ -91,7 +89,6 @@ const getFirstMatchKVCases = [ sourceKeys: 'rivals.1', }, expectedOutput: { - value: null, key: '', }, }, From 5ef41f585b458bd0fcd34a4cd3dce80d03a92b4d Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Fri, 10 Nov 2023 16:28:48 +0530 Subject: [PATCH 19/21] fix: update the default value to be returned to be undefined Signed-off-by: Sai Sankeerth --- src/v0/util/index.js | 2 +- test/utils/getFirstMatchingKeyAndValue.test.ts | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/v0/util/index.js b/src/v0/util/index.js index f9f1e2897e..46dfaa97bd 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -683,7 +683,7 @@ const getFirstMatchingKeyAndValue = (message, sourceKeys) => { } } - return { value: null, key: '' }; + return { key: '' }; }; // get a field value from message. diff --git a/test/utils/getFirstMatchingKeyAndValue.test.ts b/test/utils/getFirstMatchingKeyAndValue.test.ts index fac7ae9670..71a62c699f 100644 --- a/test/utils/getFirstMatchingKeyAndValue.test.ts +++ b/test/utils/getFirstMatchingKeyAndValue.test.ts @@ -32,7 +32,6 @@ const getFirstMatchKVCases = [ sourceKeys: ['spain.rivals', 'rivals.spain'], }, expectedOutput: { - value: null, key: '', }, }, @@ -59,7 +58,6 @@ const getFirstMatchKVCases = [ sourceKeys: 'rivals.1', }, expectedOutput: { - value: null, key: '', }, }, @@ -99,7 +97,6 @@ const getFirstMatchKVCases = [ sourceKeys: 'rivals.1', }, expectedOutput: { - value: null, key: '', }, }, From 3365332bbbdad62d04ac790a742d638cf02a9a30 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 14 Nov 2023 15:19:55 +0530 Subject: [PATCH 20/21] fix: addressing comments - 3 - move DTRequest to types/index.ts - remove unnecessary optional chaining Signed-off-by: Sai Sankeerth --- src/middlewares/enricher.ts | 3 +-- src/types/index.ts | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/middlewares/enricher.ts b/src/middlewares/enricher.ts index ba46f4d6ed..2dd7daef34 100644 --- a/src/middlewares/enricher.ts +++ b/src/middlewares/enricher.ts @@ -2,12 +2,11 @@ import { Context, Next } from 'koa'; import { ProcessorTransformationRequest, RouterTransformationRequest, + DTRequest, RouterTransformationRequestData, } from '../types'; import GeoLocationHelper from '../helpers/geoLocation'; -export type DTRequest = RouterTransformationRequest | ProcessorTransformationRequest[]; - export default class GeoEnricher { private static enrichWithGeoInfo( data: RouterTransformationRequestData[], diff --git a/src/types/index.ts b/src/types/index.ts index 3b95b5c5ec..7d11ed5147 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,5 +1,7 @@ import { CatchErr, FixMe } from '../util/types'; +export type DTRequest = RouterTransformationRequest | ProcessorTransformationRequest[]; + /* eslint-disable @typescript-eslint/no-explicit-any */ type ProcessorTransformationOutput = { version: string; From 67f9217b1f5e8fdba02edb6e6111bccaa95377e2 Mon Sep 17 00:00:00 2001 From: Sai Sankeerth Date: Tue, 14 Nov 2023 15:28:22 +0530 Subject: [PATCH 21/21] fix: expanded type name & extra optional chaining removal Signed-off-by: Sai Sankeerth --- src/helpers/geoLocation.ts | 2 +- src/middlewares/enricher.ts | 4 ++-- src/types/index.ts | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/helpers/geoLocation.ts b/src/helpers/geoLocation.ts index 4becad206a..9fa7c9c1fb 100644 --- a/src/helpers/geoLocation.ts +++ b/src/helpers/geoLocation.ts @@ -38,7 +38,7 @@ export default class GeoLocationHelper { const mappedAddress = Object.entries(addressFieldToGeoFieldMap).reduce( (agg, [addressFieldKey, geoFieldKey]) => { - if (!address?.[addressFieldKey] && msg?.context?.geo?.[geoFieldKey]) { + if (!address?.[addressFieldKey] && msg.context.geo?.[geoFieldKey]) { return { [addressFieldKey]: msg.context.geo[geoFieldKey], ...agg }; } return agg; diff --git a/src/middlewares/enricher.ts b/src/middlewares/enricher.ts index 2dd7daef34..a9a92a8b6d 100644 --- a/src/middlewares/enricher.ts +++ b/src/middlewares/enricher.ts @@ -2,7 +2,7 @@ import { Context, Next } from 'koa'; import { ProcessorTransformationRequest, RouterTransformationRequest, - DTRequest, + DestinationTransformationRequest, RouterTransformationRequestData, } from '../types'; import GeoLocationHelper from '../helpers/geoLocation'; @@ -27,7 +27,7 @@ export default class GeoEnricher { public static async enrich(ctx: Context, next: Next) { const transformationRequest = ctx.request.body; - let transformationReq: DTRequest; + let transformationReq: DestinationTransformationRequest; let reqBody: unknown; const isRouterTransform = Array.isArray( (transformationRequest as RouterTransformationRequest)?.input, diff --git a/src/types/index.ts b/src/types/index.ts index 7d11ed5147..8ec47bc7a4 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,6 +1,8 @@ import { CatchErr, FixMe } from '../util/types'; -export type DTRequest = RouterTransformationRequest | ProcessorTransformationRequest[]; +export type DestinationTransformationRequest = + | RouterTransformationRequest + | ProcessorTransformationRequest[]; /* eslint-disable @typescript-eslint/no-explicit-any */ type ProcessorTransformationOutput = {