From 81f7cb763611e0c05b77330b16ea1905a1a33509 Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Fri, 29 Sep 2023 19:54:04 +0530 Subject: [PATCH] chore: adding support for pii filters --- src/v0/destinations/ga4/transform.js | 4 +- src/v0/destinations/ga4/utils.js | 24 ++- src/v0/destinations/ga4/utils.test.js | 284 +++++++++++++++++++++++--- 3 files changed, 268 insertions(+), 44 deletions(-) diff --git a/src/v0/destinations/ga4/transform.js b/src/v0/destinations/ga4/transform.js index dc89927085..6e591d78ff 100644 --- a/src/v0/destinations/ga4/transform.js +++ b/src/v0/destinations/ga4/transform.js @@ -106,7 +106,7 @@ const responseBuilder = (message, { Config }) => { } break; default: - throw ConfigurationError('Invalid type of client'); + throw new ConfigurationError('Invalid type of client'); } let payload = {}; @@ -233,7 +233,7 @@ const responseBuilder = (message, { Config }) => { } // Prepare GA4 user properties - const userProperties = prepareUserProperties(message); + const userProperties = prepareUserProperties(message,Config.piiPropertiesToIgnore); if (!isEmptyObject(userProperties)) { rawPayload.user_properties = userProperties; } diff --git a/src/v0/destinations/ga4/utils.js b/src/v0/destinations/ga4/utils.js index b86d14ef1b..2cd69070fc 100644 --- a/src/v0/destinations/ga4/utils.js +++ b/src/v0/destinations/ga4/utils.js @@ -166,13 +166,15 @@ const GA4_ITEM_EXCLUSION = [ ]; /** - * Remove arrays and objects from transformed payload - * @param {*} params - * @returns + * Remove arrays and objects from transformed payload + * @param {*} params + * @returns */ const removeInvalidParams = (params) => Object.fromEntries( - Object.entries(params).filter(([key, value]) => key === 'items' || (typeof value !== 'object' && !isEmpty(value))), + Object.entries(params).filter( + ([key, value]) => key === 'items' || (typeof value !== 'object' && !isEmpty(value)), + ), ); /** @@ -244,10 +246,10 @@ const getItem = (message, isItemsRequired) => { /** * Returns items array for ga4 event payload - * @param {*} message - * @param {*} item - * @param {*} itemList - * @returns + * @param {*} message + * @param {*} item + * @param {*} itemList + * @returns */ const getItemsArray = (message, item, itemList) => { let items = []; @@ -269,7 +271,7 @@ const getItemsArray = (message, item, itemList) => { } return { items, mapRootLevelPropertiesToGA4ItemsArray }; -} +}; /** * get exclusion list for a particular event * ga4ExclusionList contains the sourceKeys that are already mapped @@ -399,12 +401,12 @@ const isValidUserProperty = (key, value) => { * Function to validate and prepare user_properties * @param {*} message */ -const prepareUserProperties = (message) => { +const prepareUserProperties = (message, piiPropertiesToIgnore = []) => { const userProperties = extractCustomFields( message, {}, ['properties.user_properties', 'context.traits'], - GA4_RESERVED_USER_PROPERTY_EXCLUSION, + [...GA4_RESERVED_USER_PROPERTY_EXCLUSION, ...piiPropertiesToIgnore], ); const validatedUserProperties = Object.entries(userProperties) diff --git a/src/v0/destinations/ga4/utils.test.js b/src/v0/destinations/ga4/utils.test.js index a39fef6f5e..cdfe18396f 100644 --- a/src/v0/destinations/ga4/utils.test.js +++ b/src/v0/destinations/ga4/utils.test.js @@ -2,7 +2,7 @@ const { validateEventName, prepareUserProperties, removeInvalidParams } = requir const userPropertyData = [ { - description: "Should validate and prepare user_properties", + description: 'Should validate and prepare user_properties', input: { userId: 'user@1', group_id: 'group@1', @@ -163,39 +163,261 @@ describe('Google Analytics 4 utils test', () => { }); describe('prepareUserProperties function tests', () => { - userPropertyData.forEach((dataPoint) => { - it(`${dataPoint.description}`, () => { - try { - const output = prepareUserProperties({ context: { traits: dataPoint.input } }); - expect(output).toEqual(dataPoint.output); - } catch (error) { - expect(error.message).toEqual(dataPoint.output.error); - } + describe('prepareUserProperties', () => { + // Empty message and context returns empty object + it('should return empty object when message and context are empty', () => { + // Arrange + const message = {}; + const context = {}; + + // Act + const result = prepareUserProperties(message, []); + + // Assert + expect(result).toEqual({}); + }); + + // Filters out reserved and PII properties + it('should filter out reserved and PII properties', () => { + // Arrange + const message = { + context: { + traits: { + property3: 'value3', + property4: 'value4', + pii_property3: 'pii_value3', + pii_property4: 'pii_value4', + }, + }, + properties: { + user_properties: { + property1: 'value1', + property2: 'value2', + pii_property1: 'pii_value1', + pii_property2: 'pii_value2', + }, + }, + }; + + const piiPropertiesToIgnore = [ + 'pii_property1', + 'pii_property2', + 'pii_property3', + 'pii_property4', + ]; + + // Act + const result = prepareUserProperties(message, piiPropertiesToIgnore); + + // Assert + expect(result).toEqual({ + property1: { value: 'value1' }, + property2: { value: 'value2' }, + property3: { value: 'value3' }, + property4: { value: 'value4' }, + }); + }); + + // Validates user properties and returns them in expected format + it('should validate user properties and return them in expected format', () => { + // Arrange + const message = { + context: { + traits: { + valid_property3: 'value3', + _invalid_property3: '12_invalid_value3', + valid_property4: 'value4', + invalid_property4: [], + }, + }, + properties: { + user_properties: { + valid_property1: 'value1', + '12invalid_property1': 'ga_invalid_value1', + valid_property2: 'value2', + ga_invalid_property2: 'google_invalid_value2', + }, + }, + }; + // Act + const result = prepareUserProperties(message, []); + + // Assert + expect(result).toEqual({ + valid_property1: { value: 'value1' }, + valid_property2: { value: 'value2' }, + valid_property3: { value: 'value3' }, + valid_property4: { value: 'value4' }, + }); + }); + + // Invalid user properties are filtered out + + // User properties with invalid value types are filtered out + it('should filter out user properties with invalid value types', () => { + // Arrange + const message = { + context: { + traits: { + valid_property3: 'value3', + invalid_property3: { 456: 'value3' }, + valid_property4: 'value4', + invalid_property4: '01234567890123456789012345678901234567890123456789', + }, + }, + properties: { + user_properties: { + valid_property1: 'value1', + invalid_property1: [123, 456], + valid_property2: 'value2', + }, + }, + }; + + // Act + const result = prepareUserProperties(message, []); + + // Assert + expect(result).toEqual({ + valid_property1: { value: 'value1' }, + valid_property2: { value: 'value2' }, + valid_property3: { value: 'value3' }, + valid_property4: { value: 'value4' }, + }); + }); + + // PII properties are filtered out + it('should filter out PII properties from user_properties', () => { + // Arrange + const message = { + properties: { + user_properties: { + property1: 'value1', + property2: 'value2', + pii_property1: 'pii_value1', + pii_property2: 'pii_value2', + }, + }, + }; + const piiPropertiesToIgnore = ['pii_property1', 'pii_property2']; + + // Act + const result = prepareUserProperties(message, piiPropertiesToIgnore); + + // Assert + expect(result).toEqual({ + property1: { value: 'value1' }, + property2: { value: 'value2' }, + }); + }); + + // User properties with valid keys and values are returned in expected format + it('should return user properties with valid keys and values in expected format', () => { + // Arrange + const message = { + properties: { + user_properties: { + property1: 'value1', + property2: 'value2', + }, + }, + }; + + // Act + const result = prepareUserProperties(message, []); + + // Assert + expect(result).toEqual({ + property1: { value: 'value1' }, + property2: { value: 'value2' }, + }); + }); + + // User properties with valid keys but invalid values are filtered out + it('should filter out user properties with invalid values', () => { + // Arrange + const message = { + properties: { + user_properties: { + validKey1: 'validValue1', + validKey2: 'validValue2', + invalidKey1: '', + invalidKey2: + 'invalidValueThatIsTooLongInvalidValueThatIsTooLongInvalidValueThatIsTooLongInvalidValueThatIsTooLong', + validKey4: true, + }, + }, + }; + const piiPropertiesToIgnore = []; + + // Act + const result = prepareUserProperties(message, piiPropertiesToIgnore); + + // Assert + expect(result).toEqual({ + validKey1: { value: 'validValue1' }, + validKey2: { value: 'validValue2' }, + validKey4: { value: true }, + }); + }); + // User properties with keys starting with reserved prefixes are filtered out + it('should filter out user properties with keys starting with reserved prefixes', () => { + // Arrange + const message = { + properties: { + user_properties: { + google_property: 'value1', + ga_property: 'value2', + firebase_property: 'value3', + valid_property: 'value4', + }, + }, + }; + const piiPropertiesToIgnore = []; + + // Act + const result = prepareUserProperties(message, piiPropertiesToIgnore); + + // Assert + expect(result).toEqual({ + valid_property: { value: 'value4' }, + }); + }); + + userPropertyData.forEach((dataPoint) => { + it(`${dataPoint.description}`, () => { + try { + const output = prepareUserProperties({ context: { traits: dataPoint.input } }); + expect(output).toEqual(dataPoint.output); + } catch (error) { + expect(error.message).toEqual(dataPoint.output.error); + } + }); }); }); - }); - describe('removeInvalidValues function tests', () => { - it('Should remove empty values except for "items"', () => { - const params = { - name: 'John', - age: 30, - email: '', - city: null, - items: [], - address: {}, - phone: '123456789', - }; - - const expected = { - name: 'John', - items: [], - age: 30, - phone: '123456789', - }; - - const result = removeInvalidParams(params); - expect(result).toEqual(expected); + describe('removeInvalidValues function tests', () => { + it('Should remove empty values except for "items"', () => { + const params = { + name: 'John', + age: 30, + email: '', + city: null, + items: [], + address: {}, + phone: '123456789', + }; + + const expected = { + name: 'John', + items: [], + age: 30, + phone: '123456789', + }; + + const result = removeInvalidParams(params); + expect(result).toEqual(expected); + }); }); }); });