diff --git a/CHANGELOG.md b/CHANGELOG.md index c860747b6f..854f45b096 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.79.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.79.0...v1.79.1) (2024-09-24) + + +### Bug Fixes + +* allow users context traits and underscore divide numbers configuration ([#3752](https://github.com/rudderlabs/rudder-transformer/issues/3752)) ([386d2ab](https://github.com/rudderlabs/rudder-transformer/commit/386d2ab88c0fe72dc47ba119be08ad1c0cd6d51b)) +* populate users fields for sentAt, timestamp and originalTimestamp ([#3753](https://github.com/rudderlabs/rudder-transformer/issues/3753)) ([f50effe](https://github.com/rudderlabs/rudder-transformer/commit/f50effeeabdb888f82451c225a80971dbe6532b6)) +* prefer event check vs config check for vdm ([#3754](https://github.com/rudderlabs/rudder-transformer/issues/3754)) ([b2c1a18](https://github.com/rudderlabs/rudder-transformer/commit/b2c1a1893dfb957ac7a24c000b33cd254ef54b6c)) +* support different lookup fields and custom_attributes for rETL events ([#3751](https://github.com/rudderlabs/rudder-transformer/issues/3751)) ([10d914e](https://github.com/rudderlabs/rudder-transformer/commit/10d914e25203bd6ae95801c2a98c17690bd2d6ef)) + ## [1.79.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.78.0...v1.79.0) (2024-09-20) diff --git a/package-lock.json b/package-lock.json index 98bc41379e..ae018a5ec9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.79.0", + "version": "1.79.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.79.0", + "version": "1.79.1", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index c598db805c..ede36441eb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.79.0", + "version": "1.79.1", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { diff --git a/src/cdk/v2/destinations/intercom/procWorkflow.yaml b/src/cdk/v2/destinations/intercom/procWorkflow.yaml index 0f2ac18fbc..db1ed02d57 100644 --- a/src/cdk/v2/destinations/intercom/procWorkflow.yaml +++ b/src/cdk/v2/destinations/intercom/procWorkflow.yaml @@ -77,8 +77,8 @@ steps: template: | const payload = .message.context.mappedToDestination ? $.outputs.rEtlPayload : $.outputs.identifyTransformationForLatestVersion; payload.name = $.getName(.message); - payload.custom_attributes = .message.context.traits || {}; - payload.custom_attributes = $.filterCustomAttributes(payload, "user", .destination); + payload.custom_attributes = (.message.context.mappedToDestination ? .message.traits.custom_attributes : .message.context.traits) || {}; + payload.custom_attributes = $.filterCustomAttributes(payload, "user", .destination, .message); payload.external_id = !payload.external_id && .destination.Config.sendAnonymousId && .message.anonymousId ? .message.anonymousId : payload.external_id; $.context.payload = payload; $.assert($.context.payload.external_id || $.context.payload.email, "Either email or userId is required for Identify call"); @@ -114,7 +114,7 @@ steps: update_last_request_at: typeof .destination.Config.updateLastRequestAt === 'boolean' ? .destination.Config.updateLastRequestAt : true } payload.companies = $.getCompaniesList(payload); - payload.custom_attributes = !.message.context.mappedToDestination ? $.filterCustomAttributes(payload, "user", .destination); + payload.custom_attributes = !.message.context.mappedToDestination ? $.filterCustomAttributes(payload, "user", .destination,.message); payload.user_id = !payload.user_id && .destination.Config.sendAnonymousId && .message.anonymousId ? .message.anonymousId : payload.user_id; $.context.payload = payload; $.assert($.context.payload.user_id || $.context.payload.email, "Either of `email` or `userId` is required for Identify call"); @@ -175,7 +175,7 @@ steps: $.assert(.message.groupId, "groupId is required for group call"); const payload = .message.context.mappedToDestination ? $.outputs.rEtlPayload : $.outputs.groupTransformation; payload.custom_attributes = .message.traits || {}; - payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination); + payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination,.message); $.context.payload = payload; - name: whenSearchContactFound condition: $.isDefinedAndNotNull($.outputs.searchContact) @@ -214,7 +214,7 @@ steps: ...payload, custom_attributes : $.getFieldValueFromMessage(.message, "traits") || {} } - payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination); + payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination, .message); response.body.JSON = $.removeUndefinedAndNullValues(payload); response.endpoint = $.getBaseEndpoint(.destination) + "/" + "companies"; response.headers = $.getHeaders(.destination, $.outputs.apiVersion); diff --git a/src/cdk/v2/destinations/intercom/utils.js b/src/cdk/v2/destinations/intercom/utils.js index dc483e040b..22af726e84 100644 --- a/src/cdk/v2/destinations/intercom/utils.js +++ b/src/cdk/v2/destinations/intercom/utils.js @@ -233,20 +233,26 @@ const attachUserAndCompany = (message, Config) => { * @param {*} type * @returns */ -const filterCustomAttributes = (payload, type, destination) => { +const filterCustomAttributes = (payload, type, destination, message) => { let ReservedAttributesList; let { apiVersion } = destination.Config; apiVersion = isDefinedAndNotNull(apiVersion) ? apiVersion : 'v2'; + // we are discarding the lookup field from custom attributes + const lookupField = getLookUpField(message); if (type === 'user') { - ReservedAttributesList = - apiVersion === 'v1' + ReservedAttributesList = [ + ...(apiVersion === 'v1' ? ReservedAttributes.v1UserAttributes - : ReservedAttributes.v2UserAttributes; + : ReservedAttributes.v2UserAttributes), + lookupField, + ]; } else { - ReservedAttributesList = - apiVersion === 'v1' + ReservedAttributesList = [ + ...(apiVersion === 'v1' ? ReservedAttributes.v1CompanyAttributes - : ReservedAttributes.v2CompanyAttributes; + : ReservedAttributes.v2CompanyAttributes), + lookupField !== 'email' && lookupField, + ]; } let customAttributes = { ...get(payload, 'custom_attributes') }; if (customAttributes) { @@ -270,7 +276,10 @@ const filterCustomAttributes = (payload, type, destination) => { */ const searchContact = async (message, destination, metadata) => { const lookupField = getLookUpField(message); - const lookupFieldValue = getFieldValueFromMessage(message, lookupField); + let lookupFieldValue = getFieldValueFromMessage(message, lookupField); + if (!lookupFieldValue) { + lookupFieldValue = message?.context?.traits?.[lookupField]; + } const data = JSON.stringify({ query: { operator: 'AND', diff --git a/src/v0/destinations/fb_custom_audience/recordTransform.js b/src/v0/destinations/fb_custom_audience/recordTransform.js index 9f48a37fca..db1fbeec59 100644 --- a/src/v0/destinations/fb_custom_audience/recordTransform.js +++ b/src/v0/destinations/fb_custom_audience/recordTransform.js @@ -269,10 +269,11 @@ const processRecordInputsV2 = (groupedRecordInputs) => { function processRecordInputs(groupedRecordInputs) { const event = groupedRecordInputs[0]; - if (isEventSentByVDMV2Flow(event)) { - return processRecordInputsV2(groupedRecordInputs); + // First check for rETL flow and second check for ES flow + if (isEventSentByVDMV1Flow(event) || !isEventSentByVDMV2Flow(event)) { + return processRecordInputsV1(groupedRecordInputs); } - return processRecordInputsV1(groupedRecordInputs); + return processRecordInputsV2(groupedRecordInputs); } module.exports = { diff --git a/src/warehouse/index.js b/src/warehouse/index.js index 3c2b04079d..4afa8f72c2 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -643,6 +643,18 @@ function processWarehouseMessage(message, options) { const skipReservedKeywordsEscaping = options.integrationOptions.skipReservedKeywordsEscaping || false; + // underscoreDivideNumbers when set to false, if a column has a format like "_v_3_", it will be formatted to "_v3_" + // underscoreDivideNumbers when set to true, if a column has a format like "_v_3_", we keep it like that + // For older destinations, it will come as true and for new destinations this config will not be present which means we will treat it as false. + options.underscoreDivideNumbers = options.destConfig?.underscoreDivideNumbers || false; + + // allowUsersContextTraits when set to true, if context.traits.* is present, it will be added as context_traits_* and *, + // e.g., for context.traits.name, context_traits_name and name will be added to the user's table. + // allowUsersContextTraits when set to false, if context.traits.* is present, it will be added only as context_traits_* + // e.g., for context.traits.name, only context_traits_name will be added to the user's table. + // For older destinations, it will come as true, and for new destinations this config will not be present, which means we will treat it as false. + const allowUsersContextTraits = options.destConfig?.allowUsersContextTraits || false; + addJsonKeysToOptions(options); if (isBlank(message.messageId)) { @@ -898,16 +910,18 @@ function processWarehouseMessage(message, options) { `${eventType + '_userProperties_'}`, 2, ); - setDataFromInputAndComputeColumnTypes( - utils, - eventType, - commonProps, - message.context ? message.context.traits : {}, - commonColumnTypes, - options, - `${eventType + '_context_traits_'}`, - 3, - ); + if (allowUsersContextTraits) { + setDataFromInputAndComputeColumnTypes( + utils, + eventType, + commonProps, + message.context ? message.context.traits : {}, + commonColumnTypes, + options, + `${eventType + '_context_traits_'}`, + 3, + ); + } setDataFromInputAndComputeColumnTypes( utils, eventType, @@ -987,11 +1001,23 @@ function processWarehouseMessage(message, options) { const usersEvent = { ...commonProps }; const usersColumnTypes = {}; + let userColumnMappingRules = whUserColumnMappingRules; + if (!isDataLakeProvider(options.provider)) { + userColumnMappingRules = { + ...userColumnMappingRules, + ...{ + sent_at: 'sentAt', + timestamp: 'timestamp', + original_timestamp: 'originalTimestamp', + }, + }; + } + setDataFromColumnMappingAndComputeColumnTypes( utils, usersEvent, message, - whUserColumnMappingRules, + userColumnMappingRules, usersColumnTypes, options, ); diff --git a/src/warehouse/snakecase/snakecase.js b/src/warehouse/snakecase/snakecase.js new file mode 100644 index 0000000000..1e6586e7f2 --- /dev/null +++ b/src/warehouse/snakecase/snakecase.js @@ -0,0 +1,37 @@ +const { toString } = require('lodash'); +const { unicodeWords, unicodeWordsWithNumbers } = require('./unicodeWords'); + +const hasUnicodeWord = RegExp.prototype.test.bind( + /[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/, +); + +/** Used to match words composed of alphanumeric characters. */ +// eslint-disable-next-line no-control-regex +const reAsciiWord = /[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g; + +function asciiWords(string) { + return string.match(reAsciiWord); +} + +function words(string) { + const result = hasUnicodeWord(string) ? unicodeWords(string) : asciiWords(string); + return result || []; +} + +function wordsWithNumbers(string) { + const result = hasUnicodeWord(string) ? unicodeWordsWithNumbers(string) : asciiWords(string); + return result || []; +} + +const snakeCase = (string) => + words(toString(string).replace(/['\u2019]/g, '')).reduce( + (result, word, index) => result + (index ? '_' : '') + word.toLowerCase(), + '', + ); +const snakeCaseWithNumbers = (string) => + wordsWithNumbers(toString(string).replace(/['\u2019]/g, '')).reduce( + (result, word, index) => result + (index ? '_' : '') + word.toLowerCase(), + '', + ); + +module.exports = { words, wordsWithNumbers, snakeCase, snakeCaseWithNumbers }; diff --git a/src/warehouse/snakecase/unicodeWords.js b/src/warehouse/snakecase/unicodeWords.js new file mode 100644 index 0000000000..d7b15806c7 --- /dev/null +++ b/src/warehouse/snakecase/unicodeWords.js @@ -0,0 +1,94 @@ +/** Used to compose unicode character classes. */ +const rsAstralRange = '\\ud800-\\udfff'; +const rsComboMarksRange = '\\u0300-\\u036f'; +const reComboHalfMarksRange = '\\ufe20-\\ufe2f'; +const rsComboSymbolsRange = '\\u20d0-\\u20ff'; +const rsComboMarksExtendedRange = '\\u1ab0-\\u1aff'; +const rsComboMarksSupplementRange = '\\u1dc0-\\u1dff'; +const rsComboRange = + rsComboMarksRange + + reComboHalfMarksRange + + rsComboSymbolsRange + + rsComboMarksExtendedRange + + rsComboMarksSupplementRange; +const rsDingbatRange = '\\u2700-\\u27bf'; +const rsLowerRange = 'a-z\\xdf-\\xf6\\xf8-\\xff'; +const rsMathOpRange = '\\xac\\xb1\\xd7\\xf7'; +const rsNonCharRange = '\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf'; +const rsPunctuationRange = '\\u2000-\\u206f'; +const rsSpaceRange = + ' \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000'; +const rsUpperRange = 'A-Z\\xc0-\\xd6\\xd8-\\xde'; +const rsVarRange = '\\ufe0e\\ufe0f'; +const rsBreakRange = rsMathOpRange + rsNonCharRange + rsPunctuationRange + rsSpaceRange; + +/** Used to compose unicode capture groups. */ +const rsApos = "['\u2019]"; +const rsBreak = `[${rsBreakRange}]`; +const rsCombo = `[${rsComboRange}]`; +const rsDigit = '\\d'; +const rsDingbat = `[${rsDingbatRange}]`; +const rsLower = `[${rsLowerRange}]`; +const rsMisc = `[^${rsAstralRange}${rsBreakRange + rsDigit + rsDingbatRange + rsLowerRange + rsUpperRange}]`; +const rsFitz = '\\ud83c[\\udffb-\\udfff]'; +const rsModifier = `(?:${rsCombo}|${rsFitz})`; +const rsNonAstral = `[^${rsAstralRange}]`; +const rsRegional = '(?:\\ud83c[\\udde6-\\uddff]){2}'; +const rsSurrPair = '[\\ud800-\\udbff][\\udc00-\\udfff]'; +const rsUpper = `[${rsUpperRange}]`; +const rsZWJ = '\\u200d'; + +/** Used to compose unicode regexes. */ +const rsMiscLower = `(?:${rsLower}|${rsMisc})`; +const rsMiscUpper = `(?:${rsUpper}|${rsMisc})`; +const rsOptContrLower = `(?:${rsApos}(?:d|ll|m|re|s|t|ve))?`; +const rsOptContrUpper = `(?:${rsApos}(?:D|LL|M|RE|S|T|VE))?`; +const reOptMod = `${rsModifier}?`; +const rsOptVar = `[${rsVarRange}]?`; +const rsOptJoin = `(?:${rsZWJ}(?:${[rsNonAstral, rsRegional, rsSurrPair].join('|')})${rsOptVar + reOptMod})*`; +const rsOrdLower = '\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])'; +const rsOrdUpper = '\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])'; +const rsSeq = rsOptVar + reOptMod + rsOptJoin; +const rsEmoji = `(?:${[rsDingbat, rsRegional, rsSurrPair].join('|')})${rsSeq}`; + +const reUnicodeWords = RegExp( + [ + `${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`, // Regular words, lowercase letters followed by optional contractions + `${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`, // Miscellaneous uppercase characters with optional contractions + `${rsUpper}?${rsMiscLower}+${rsOptContrLower}`, // Miscellaneous lowercase sequences with optional contractions + `${rsUpper}+${rsOptContrUpper}`, // All uppercase words with optional contractions (e.g., "THIS") + rsOrdUpper, // Ordinals for uppercase (e.g., "1ST", "2ND") + rsOrdLower, // Ordinals for lowercase (e.g., "1st", "2nd") + `${rsDigit}+`, // Pure digits (e.g., "123") + rsEmoji, // Emojis (e.g., 😀, ❤️) + ].join('|'), + 'g', +); + +const reUnicodeWordsWithNumbers = RegExp( + [ + `${rsUpper}?${rsLower}+${rsDigit}+`, // Lowercase letters followed by digits (e.g., "abc123") + `${rsUpper}+${rsDigit}+`, // Uppercase letters followed by digits (e.g., "ABC123") + `${rsDigit}+${rsUpper}?${rsLower}+`, // Digits followed by lowercase letters (e.g., "123abc") + `${rsDigit}+${rsUpper}+`, // Digits followed by uppercase letters (e.g., "123ABC") + `${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`, // Regular words, lowercase letters followed by optional contractions + `${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`, // Miscellaneous uppercase characters with optional contractions + `${rsUpper}?${rsMiscLower}+${rsOptContrLower}`, // Miscellaneous lowercase sequences with optional contractions + `${rsUpper}+${rsOptContrUpper}`, // All uppercase words with optional contractions (e.g., "THIS") + rsOrdUpper, // Ordinals for uppercase (e.g., "1ST", "2ND") + rsOrdLower, // Ordinals for lowercase (e.g., "1st", "2nd") + `${rsDigit}+`, // Pure digits (e.g., "123") + rsEmoji, // Emojis (e.g., 😀, ❤️) + ].join('|'), + 'g', +); + +function unicodeWords(string) { + return string.match(reUnicodeWords); +} + +function unicodeWordsWithNumbers(string) { + return string.match(reUnicodeWordsWithNumbers); +} + +module.exports = { unicodeWords, unicodeWordsWithNumbers }; diff --git a/src/warehouse/util.js b/src/warehouse/util.js index b4b22721fd..7f4e224a34 100644 --- a/src/warehouse/util.js +++ b/src/warehouse/util.js @@ -1,7 +1,5 @@ -const _ = require('lodash'); const get = require('get-value'); -const v0 = require('./v0/util'); const v1 = require('./v1/util'); const { PlatformError, InstrumentationError } = require('@rudderstack/integrations-lib'); const { isBlank } = require('./config/helpers'); @@ -112,14 +110,7 @@ function validTimestamp(input) { } function getVersionedUtils(schemaVersion) { - switch (schemaVersion) { - case 'v0': - return v0; - case 'v1': - return v1; - default: - return v1; - } + return v1; } function isRudderSourcesEvent(event) { diff --git a/src/warehouse/v0/util.js b/src/warehouse/v0/util.js deleted file mode 100644 index 5917f8ea08..0000000000 --- a/src/warehouse/v0/util.js +++ /dev/null @@ -1,87 +0,0 @@ -const reservedANSIKeywordsMap = require('../config/ReservedKeywords.json'); -const { isDataLakeProvider } = require('../config/helpers'); - -const toSnakeCase = (str) => { - if (!str) { - return ''; - } - return String(str) - .replace(/^[^A-Za-z0-9]*|[^A-Za-z0-9]*$/g, '') - .replace(/([a-z])([A-Z])/g, (m, a, b) => `${a}_${b.toLowerCase()}`) - .replace(/[^A-Za-z0-9]+|_+/g, '_') - .toLowerCase(); -}; - -function toSafeDBString(provider, name = '') { - let parsedStr = name; - if (parseInt(name[0], 10) >= 0) { - parsedStr = `_${name}`; - } - parsedStr = parsedStr.replace(/[^a-zA-Z0-9_]+/g, ''); - if (isDataLakeProvider(provider)) { - return parsedStr; - } - switch (provider) { - case 'postgres': - return parsedStr.substr(0, 63); - default: - return parsedStr.substr(0, 127); - } -} - -function safeTableName(provider, name = '') { - let tableName = name; - if (tableName === '') { - tableName = 'STRINGEMPTY'; - } - if (provider === 'snowflake') { - tableName = tableName.toUpperCase(); - } - if (provider === 'rs' || isDataLakeProvider(provider)) { - tableName = tableName.toLowerCase(); - } - if (provider === 'postgres') { - tableName = tableName.substr(0, 63); - tableName = tableName.toLowerCase(); - } - if (reservedANSIKeywordsMap[provider.toUpperCase()][tableName.toUpperCase()]) { - tableName = `_${tableName}`; - } - return tableName; -} - -function safeColumnName(provider, name = '') { - let columnName = name; - if (columnName === '') { - columnName = 'STRINGEMPTY'; - } - if (provider === 'snowflake') { - columnName = columnName.toUpperCase(); - } - if (provider === 'rs' || isDataLakeProvider(provider)) { - columnName = columnName.toLowerCase(); - } - if (provider === 'postgres') { - columnName = columnName.substr(0, 63); - columnName = columnName.toLowerCase(); - } - if (reservedANSIKeywordsMap[provider.toUpperCase()][columnName.toUpperCase()]) { - columnName = `_${columnName}`; - } - return columnName; -} - -function transformTableName(name = '') { - return toSnakeCase(name); -} - -function transformColumnName(provider, name = '') { - return toSafeDBString(provider, name); -} - -module.exports = { - safeColumnName, - safeTableName, - transformColumnName, - transformTableName, -}; diff --git a/src/warehouse/v1/util.js b/src/warehouse/v1/util.js index 1c44a2385e..d1289bc674 100644 --- a/src/warehouse/v1/util.js +++ b/src/warehouse/v1/util.js @@ -1,8 +1,7 @@ -const _ = require('lodash'); - const reservedANSIKeywordsMap = require('../config/ReservedKeywords.json'); const { isDataLakeProvider } = require('../config/helpers'); const { TransformationError } = require('@rudderstack/integrations-lib'); +const { snakeCase, snakeCaseWithNumbers } = require('../snakecase/snakecase'); function safeTableName(options, name = '') { const { provider } = options; @@ -82,7 +81,7 @@ function safeColumnName(options, name = '') { path to $1,00,000 to path_to_1_00_000 return an empty string if it couldn't find a char if its ascii value doesnt belong to numbers or english alphabets */ -function transformName(provider, name = '') { +function transformName(options, provider, name = '') { const extractedValues = []; let extractedValue = ''; for (let i = 0; i < name.length; i += 1) { @@ -104,14 +103,17 @@ function transformName(provider, name = '') { if (extractedValue !== '') { extractedValues.push(extractedValue); } + const underscoreDivideNumbers = options?.underscoreDivideNumbers || false; + const snakeCaseFn = underscoreDivideNumbers ? snakeCase : snakeCaseWithNumbers; + let key = extractedValues.join('_'); if (name.startsWith('_')) { // do not remove leading underscores to allow esacaping rudder keywords with underscore // _timestamp -> _timestamp // __timestamp -> __timestamp - key = name.match(/^_*/)[0] + _.snakeCase(key.replace(/^_*/, '')); + key = name.match(/^_*/)[0] + snakeCaseFn(key.replace(/^_*/, '')); } else { - key = _.snakeCase(key); + key = snakeCaseFn(key); } if (key !== '' && key.charCodeAt(0) >= 48 && key.charCodeAt(0) <= 57) { @@ -150,7 +152,7 @@ function toBlendoCase(name = '') { function transformTableName(options, name = '') { const useBlendoCasing = options.integrationOptions?.useBlendoCasing || false; - return useBlendoCasing ? toBlendoCase(name) : transformName('', name); + return useBlendoCasing ? toBlendoCase(name) : transformName(options, '', name); } function transformColumnName(options, name = '') { @@ -158,7 +160,7 @@ function transformColumnName(options, name = '') { const useBlendoCasing = options.integrationOptions?.useBlendoCasing || false; return useBlendoCasing ? transformNameToBlendoCase(provider, name) - : transformName(provider, name); + : transformName(options, provider, name); } module.exports = { diff --git a/test/__tests__/data/warehouse/events.js b/test/__tests__/data/warehouse/events.js index ef9cc21096..bca6f776be 100644 --- a/test/__tests__/data/warehouse/events.js +++ b/test/__tests__/data/warehouse/events.js @@ -8,7 +8,9 @@ const sampleEvents = { Config: { restApiKey: "dummyApiKey", prefixProperties: true, - useNativeSDK: false + useNativeSDK: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, DestinationDefinition: { DisplayName: "Braze", @@ -514,7 +516,9 @@ const sampleEvents = { Config: { restApiKey: "dummyApiKey", prefixProperties: true, - useNativeSDK: false + useNativeSDK: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, DestinationDefinition: { DisplayName: "Braze", @@ -1026,7 +1030,9 @@ const sampleEvents = { Config: { restApiKey: "dummyApiKey", prefixProperties: true, - useNativeSDK: false + useNativeSDK: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, DestinationDefinition: { DisplayName: "Braze", @@ -1283,7 +1289,9 @@ const sampleEvents = { mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, - trackNamedPages: false + trackNamedPages: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, Enabled: true }, @@ -1532,7 +1540,9 @@ const sampleEvents = { mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, - trackNamedPages: false + trackNamedPages: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, Enabled: true }, @@ -1752,7 +1762,9 @@ const sampleEvents = { Config: { apiKey: "dummyApiKey", prefixProperties: true, - useNativeSDK: false + useNativeSDK: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, DestinationDefinition: { DisplayName: "Kiss Metrics", @@ -2016,7 +2028,9 @@ const sampleEvents = { Config: { restApiKey: "dummyApiKey", prefixProperties: true, - useNativeSDK: false + useNativeSDK: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, DestinationDefinition: { DisplayName: "Braze", diff --git a/test/__tests__/data/warehouse/integration_options_events.js b/test/__tests__/data/warehouse/integration_options_events.js index 05a2d51abd..35eafb6a9b 100644 --- a/test/__tests__/data/warehouse/integration_options_events.js +++ b/test/__tests__/data/warehouse/integration_options_events.js @@ -20,7 +20,9 @@ const sampleEvents = { input: { destination: { Config: { - jsonPaths: " testMap.nestedMap, testArray" + jsonPaths: " testMap.nestedMap, testArray", + allowUsersContextTraits: true, + underscoreDivideNumbers: true } }, message: { @@ -731,7 +733,10 @@ const sampleEvents = { users: { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { type: "identify", @@ -880,6 +885,9 @@ const sampleEvents = { "email": "user123@email.com", "id": "user123", "phone": "+917836362334", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "received_at": "2020-01-24T06:29:02.403Z" }, "metadata": { @@ -906,6 +914,9 @@ const sampleEvents = { "id": "string", "phone": "string", "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -1074,6 +1085,9 @@ const sampleEvents = { "email": "user123@email.com", "id": "user123", "phone": "+917836362334", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "received_at": "2020-01-24T06:29:02.403Z" }, "metadata": { @@ -1101,6 +1115,9 @@ const sampleEvents = { "loaded_at": "datetime", "phone": "string", "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -1199,6 +1216,9 @@ const sampleEvents = { "EMAIL": "user123@email.com", "ID": "user123", "PHONE": "+917836362334", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", "RECEIVED_AT": "2020-01-24T06:29:02.403Z" }, "metadata": { @@ -1225,6 +1245,9 @@ const sampleEvents = { "ID": "string", "PHONE": "string", "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "ORIGINAL_TIMESTAMP": "datetime", "UUID_TS": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -1355,6 +1378,254 @@ const sampleEvents = { "table": "users" } } + ], + gcs_datalake: [ + { + "data": { + "timestamp": "2020-01-24T06:29:02.403Z", + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "user_id": "user123" + }, + "metadata": { + "columns": { + "timestamp": "datetime", + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + azure_datalake: [ + { + "data": { + "timestamp": "2020-01-24T06:29:02.403Z", + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "user_id": "user123" + }, + "metadata": { + "columns": { + "timestamp": "datetime", + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } ] } } @@ -1374,6 +1645,18 @@ function opOutput(eventType, provider) { return _.cloneDeep(sampleEvents[eventType].output.rs); case "bq": return _.cloneDeep(sampleEvents[eventType].output.bq); + case "gcs_datalake": + if (eventType === 'users') { + return _.cloneDeep(sampleEvents[eventType].output.gcs_datalake); + } else { + return _.cloneDeep(sampleEvents[eventType].output.default); + } + case "azure_datalake": + if (eventType === 'users') { + return _.cloneDeep(sampleEvents[eventType].output.azure_datalake); + } else { + return _.cloneDeep(sampleEvents[eventType].output.default); + } default: return _.cloneDeep(sampleEvents[eventType].output.default); } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js index ee748a1a2b..30cce51fb7 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js index 906c3ada23..9d38ecc292 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js index 13b243f3a7..bbae993b27 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js index 47b7f1209a..42f5cccf49 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { type: "identify", @@ -230,6 +233,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -261,6 +267,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -374,6 +383,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -405,6 +417,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -519,6 +534,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -551,6 +569,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -664,6 +685,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -695,6 +719,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -808,6 +835,9 @@ module.exports = { "PHONE": "+917836362334", "RECEIVED_AT": "2020-01-24T06:29:02.403Z", "T_MAP_NESTED_MAP_N_1": "nested prop 1", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", "UP_MAP_NESTED_MAP_N_1": "nested prop 1" }, "metadata": { @@ -839,6 +869,9 @@ module.exports = { "RECEIVED_AT": "datetime", "T_MAP_NESTED_MAP_N_1": "string", "UP_MAP_NESTED_MAP_N_1": "string", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "ORIGINAL_TIMESTAMP": "datetime", "UUID_TS": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -846,5 +879,149 @@ module.exports = { } } ], + datalake: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js index 0aac8a3c23..dc2d3a3318 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js index bad325c908..9b789d2747 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js index 5070284e99..691ce57ca1 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js index 91f19c5c77..d5d57f85b9 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js index 7a36e4787e..a950b7f526 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js index f84f9d33ed..6979aa1100 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js index 3d6164b430..89fcc23cd5 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { type: "identify", @@ -230,6 +233,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -261,6 +267,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -373,6 +382,9 @@ module.exports = { "id": "user123", "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" }, @@ -405,6 +417,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map": "json", "up_map_nested_map": "json", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -518,6 +533,9 @@ module.exports = { "id": "user123", "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" }, @@ -551,6 +569,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map": "string", "up_map_nested_map": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -663,6 +684,9 @@ module.exports = { "id": "user123", "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" }, @@ -695,6 +719,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map": "json", "up_map_nested_map": "json", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -807,6 +834,9 @@ module.exports = { "ID": "user123", "PHONE": "+917836362334", "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", "T_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}" }, @@ -839,6 +869,9 @@ module.exports = { "RECEIVED_AT": "datetime", "T_MAP_NESTED_MAP": "json", "UP_MAP_NESTED_MAP": "json", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "ORIGINAL_TIMESTAMP": "datetime", "UUID_TS": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -846,5 +879,149 @@ module.exports = { } } ], + datalake: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js index 136f355b21..d5282c1cfd 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js index b11b311ebb..3eec47b89d 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js index 7ed95685ff..6b411835db 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/snakecase.test.js b/test/__tests__/snakecase.test.js new file mode 100644 index 0000000000..146e2e53c6 --- /dev/null +++ b/test/__tests__/snakecase.test.js @@ -0,0 +1,497 @@ +const _ = require('lodash'); +const {words, wordsWithNumbers, snakeCase, snakeCaseWithNumbers} = require("../../src/warehouse/snakecase/snakecase"); + +const burredLetters = [ + // Latin-1 Supplement letters. + '\xc0', + '\xc1', + '\xc2', + '\xc3', + '\xc4', + '\xc5', + '\xc6', + '\xc7', + '\xc8', + '\xc9', + '\xca', + '\xcb', + '\xcc', + '\xcd', + '\xce', + '\xcf', + '\xd0', + '\xd1', + '\xd2', + '\xd3', + '\xd4', + '\xd5', + '\xd6', + '\xd8', + '\xd9', + '\xda', + '\xdb', + '\xdc', + '\xdd', + '\xde', + '\xdf', + '\xe0', + '\xe1', + '\xe2', + '\xe3', + '\xe4', + '\xe5', + '\xe6', + '\xe7', + '\xe8', + '\xe9', + '\xea', + '\xeb', + '\xec', + '\xed', + '\xee', + '\xef', + '\xf0', + '\xf1', + '\xf2', + '\xf3', + '\xf4', + '\xf5', + '\xf6', + '\xf8', + '\xf9', + '\xfa', + '\xfb', + '\xfc', + '\xfd', + '\xfe', + '\xff', + // Latin Extended-A letters. + '\u0100', + '\u0101', + '\u0102', + '\u0103', + '\u0104', + '\u0105', + '\u0106', + '\u0107', + '\u0108', + '\u0109', + '\u010a', + '\u010b', + '\u010c', + '\u010d', + '\u010e', + '\u010f', + '\u0110', + '\u0111', + '\u0112', + '\u0113', + '\u0114', + '\u0115', + '\u0116', + '\u0117', + '\u0118', + '\u0119', + '\u011a', + '\u011b', + '\u011c', + '\u011d', + '\u011e', + '\u011f', + '\u0120', + '\u0121', + '\u0122', + '\u0123', + '\u0124', + '\u0125', + '\u0126', + '\u0127', + '\u0128', + '\u0129', + '\u012a', + '\u012b', + '\u012c', + '\u012d', + '\u012e', + '\u012f', + '\u0130', + '\u0131', + '\u0132', + '\u0133', + '\u0134', + '\u0135', + '\u0136', + '\u0137', + '\u0138', + '\u0139', + '\u013a', + '\u013b', + '\u013c', + '\u013d', + '\u013e', + '\u013f', + '\u0140', + '\u0141', + '\u0142', + '\u0143', + '\u0144', + '\u0145', + '\u0146', + '\u0147', + '\u0148', + '\u0149', + '\u014a', + '\u014b', + '\u014c', + '\u014d', + '\u014e', + '\u014f', + '\u0150', + '\u0151', + '\u0152', + '\u0153', + '\u0154', + '\u0155', + '\u0156', + '\u0157', + '\u0158', + '\u0159', + '\u015a', + '\u015b', + '\u015c', + '\u015d', + '\u015e', + '\u015f', + '\u0160', + '\u0161', + '\u0162', + '\u0163', + '\u0164', + '\u0165', + '\u0166', + '\u0167', + '\u0168', + '\u0169', + '\u016a', + '\u016b', + '\u016c', + '\u016d', + '\u016e', + '\u016f', + '\u0170', + '\u0171', + '\u0172', + '\u0173', + '\u0174', + '\u0175', + '\u0176', + '\u0177', + '\u0178', + '\u0179', + '\u017a', + '\u017b', + '\u017c', + '\u017d', + '\u017e', + '\u017f', +]; +const emojiVar = '\ufe0f'; +const flag = '\ud83c\uddfa\ud83c\uddf8'; +const heart = `\u2764${emojiVar}`; +const hearts = '\ud83d\udc95'; +const comboGlyph = `\ud83d\udc68\u200d${heart}\u200d\ud83d\udc8B\u200d\ud83d\udc68`; +const leafs = '\ud83c\udf42'; +const rocket = '\ud83d\ude80'; +const stubTrue = function () { + return true; +}; +const stubString = function () { + return ''; +}; + +describe('words', () => { + it('should match words containing Latin Unicode letters', () => { + const expected = _.map(burredLetters, (letter) => [letter]); + const actual = _.map(burredLetters, (letter) => words(letter)); + expect(actual).toEqual(expected); + }); + + it('should work with compound words', () => { + expect(words('12ft')).toEqual(['12', 'ft']); + expect(words('aeiouAreVowels')).toEqual(['aeiou', 'Are', 'Vowels']); + expect(words('enable 6h format')).toEqual(['enable', '6', 'h', 'format']); + expect(words('enable 24H format')).toEqual(['enable', '24', 'H', 'format']); + expect(words('isISO8601')).toEqual(['is', 'ISO', '8601']); + expect(words('LETTERSAeiouAreVowels')).toEqual(['LETTERS', 'Aeiou', 'Are', 'Vowels']); + expect(words('tooLegit2Quit')).toEqual(['too', 'Legit', '2', 'Quit']); + expect(words('walk500Miles')).toEqual(['walk', '500', 'Miles']); + expect(words('xhr2Request')).toEqual(['xhr', '2', 'Request']); + expect(words('XMLHttp')).toEqual(['XML', 'Http']); + expect(words('XmlHTTP')).toEqual(['Xml', 'HTTP']); + expect(words('XmlHttp')).toEqual(['Xml', 'Http']); + }); + + it('should work with compound words containing diacritical marks', () => { + expect(words('LETTERSÆiouAreVowels')).toEqual(['LETTERS', 'Æiou', 'Are', 'Vowels']); + expect(words('æiouAreVowels')).toEqual(['æiou', 'Are', 'Vowels']); + expect(words('æiou2Consonants')).toEqual(['æiou', '2', 'Consonants']); + }); + + it('should not treat contractions as separate words', () => { + const postfixes = ['d', 'll', 'm', 're', 's', 't', 've']; + + _.each(["'", '\u2019'], (apos) => { + _.times(2, (index) => { + const actual = _.map(postfixes, (postfix) => { + const string = `a b${apos}${postfix} c`; + return words(string[index ? 'toUpperCase' : 'toLowerCase']()); + }); + const expected = _.map(postfixes, (postfix) => { + const words = ['a', `b${apos}${postfix}`, 'c']; + return _.map(words, (word) => + word[index ? 'toUpperCase' : 'toLowerCase'](), + ); + }); + expect(actual).toEqual(expected); + }); + }); + }); + + it('should not treat ordinal numbers as separate words', () => { + const ordinals = ['1st', '2nd', '3rd', '4th']; + + _.times(2, (index) => { + const expected = _.map(ordinals, (ordinal) => [ + ordinal[index ? 'toUpperCase' : 'toLowerCase'](), + ]); + const actual = _.map(expected, (expectedWords) => words(expectedWords[0])); + expect(actual).toEqual(expected); + }); + }); + + it('should prevent ReDoS', () => { + const largeWordLen = 50000; + const largeWord = 'A'.repeat(largeWordLen); + const maxMs = 1000; + const startTime = _.now(); + + expect(words(`${largeWord}ÆiouAreVowels`)).toEqual([largeWord, 'Æiou', 'Are', 'Vowels']); + + const endTime = _.now(); + const timeSpent = endTime - startTime; + + expect(timeSpent).toBeLessThan(maxMs); + }); + + it('should account for astral symbols', () => { + const string = `A ${leafs}, ${comboGlyph}, and ${rocket}`; + expect(words(string)).toEqual(['A', leafs, comboGlyph, 'and', rocket]); + }); + + it('should account for regional symbols', () => { + const pair = flag.match(/\ud83c[\udde6-\uddff]/g); + const regionals = pair.join(' '); + + expect(words(flag)).toEqual([flag]); + expect(words(regionals)).toEqual([pair[0], pair[1]]); + }); + + it('should account for variation selectors', () => { + expect(words(heart)).toEqual([heart]); + }); + + it('should match lone surrogates', () => { + const pair = hearts.split(''); + const surrogates = `${pair[0]} ${pair[1]}`; + + expect(words(surrogates)).toEqual([]); + }); +}); + +describe('wordsWithoutNumbers', () => { + it('should match words containing Latin Unicode letters', () => { + const expected = _.map(burredLetters, (letter) => [letter]); + const actual = _.map(burredLetters, (letter) => wordsWithNumbers(letter)); + expect(actual).toEqual(expected); + }); + + it('should work with compound words', () => { + expect(wordsWithNumbers('12ft')).toEqual(['12ft']); + expect(wordsWithNumbers('aeiouAreVowels')).toEqual(['aeiou', 'Are', 'Vowels']); + expect(wordsWithNumbers('enable 6h format')).toEqual(['enable', '6h', 'format']); + expect(wordsWithNumbers('enable 24H format')).toEqual(['enable', '24H', 'format']); + expect(wordsWithNumbers('isISO8601')).toEqual(['is', 'ISO8601']); + expect(wordsWithNumbers('LETTERSAeiouAreVowels')).toEqual(['LETTERS', 'Aeiou', 'Are', 'Vowels']); + expect(wordsWithNumbers('tooLegit2Quit')).toEqual(['too', 'Legit2', 'Quit']); + expect(wordsWithNumbers('walk500Miles')).toEqual(['walk500', 'Miles']); + expect(wordsWithNumbers('xhr2Request')).toEqual(['xhr2', 'Request']); + expect(wordsWithNumbers('XMLHttp')).toEqual(['XML', 'Http']); + expect(wordsWithNumbers('XmlHTTP')).toEqual(['Xml', 'HTTP']); + expect(wordsWithNumbers('XmlHttp')).toEqual(['Xml', 'Http']); + }); + + it('should work with compound words containing diacritical marks', () => { + expect(wordsWithNumbers('LETTERSÆiouAreVowels')).toEqual(['LETTERS', 'Æiou', 'Are', 'Vowels']); + expect(wordsWithNumbers('æiouAreVowels')).toEqual(['æiou', 'Are', 'Vowels']); + expect(wordsWithNumbers('æiou2Consonants')).toEqual(['æiou2', 'Consonants']); + }); + + it('should not treat contractions as separate words', () => { + const postfixes = ['d', 'll', 'm', 're', 's', 't', 've']; + + _.each(["'", '\u2019'], (apos) => { + _.times(2, (index) => { + const actual = _.map(postfixes, (postfix) => { + const string = `a b${apos}${postfix} c`; + return wordsWithNumbers(string[index ? 'toUpperCase' : 'toLowerCase']()); + }); + const expected = _.map(postfixes, (postfix) => { + const words = ['a', `b${apos}${postfix}`, 'c']; + return _.map(words, (word) => + word[index ? 'toUpperCase' : 'toLowerCase'](), + ); + }); + expect(actual).toEqual(expected); + }); + }); + }); + + it('should not treat ordinal numbers as separate words', () => { + const ordinals = ['1st', '2nd', '3rd', '4th']; + + _.times(2, (index) => { + const expected = _.map(ordinals, (ordinal) => [ + ordinal[index ? 'toUpperCase' : 'toLowerCase'](), + ]); + const actual = _.map(expected, (expectedWords) => wordsWithNumbers(expectedWords[0])); + expect(actual).toEqual(expected); + }); + }); + + it('should prevent ReDoS', () => { + const largeWordLen = 50000; + const largeWord = 'A'.repeat(largeWordLen); + const maxMs = 1000; + const startTime = _.now(); + + expect(wordsWithNumbers(`${largeWord}ÆiouAreVowels`)).toEqual([largeWord, 'Æiou', 'Are', 'Vowels']); + + const endTime = _.now(); + const timeSpent = endTime - startTime; + + expect(timeSpent).toBeLessThan(maxMs); + }); + + it('should account for astral symbols', () => { + const string = `A ${leafs}, ${comboGlyph}, and ${rocket}`; + expect(wordsWithNumbers(string)).toEqual(['A', leafs, comboGlyph, 'and', rocket]); + }); + + it('should account for regional symbols', () => { + const pair = flag.match(/\ud83c[\udde6-\uddff]/g); + const regionals = pair.join(' '); + + expect(wordsWithNumbers(flag)).toEqual([flag]); + expect(wordsWithNumbers(regionals)).toEqual([pair[0], pair[1]]); + }); + + it('should account for variation selectors', () => { + expect(wordsWithNumbers(heart)).toEqual([heart]); + }); + + it('should match lone surrogates', () => { + const pair = hearts.split(''); + const surrogates = `${pair[0]} ${pair[1]}`; + + expect(wordsWithNumbers(surrogates)).toEqual([]); + }); +}); + +describe('snakeCase snakeCaseWithNumbers', () => { + const caseMethods = { + snakeCase, + snakeCaseWithNumbers, + }; + + _.each(['snakeCase', 'snakeCaseWithNumbers'], (caseName) => { + const methodName = caseName; + const func = caseMethods[methodName]; + + const strings = [ + 'foo bar', + 'Foo bar', + 'foo Bar', + 'Foo Bar', + 'FOO BAR', + 'fooBar', + '--foo-bar--', + '__foo_bar__', + ]; + + const converted = (function () { + switch (caseName) { + case 'snakeCase': + return 'foo_bar'; + case 'snakeCaseWithNumbers': + return 'foo_bar'; + } + })(); + + it(`\`_.${methodName}\` should convert \`string\` to ${caseName} case`, () => { + const actual = _.map(strings, (string) => { + const expected = caseName === 'start' && string === 'FOO BAR' ? string : converted; + return func(string) === expected; + }); + expect(actual).toEqual(_.map(strings, stubTrue)); + }); + + it(`\`_.${methodName}\` should handle double-converting strings`, () => { + const actual = _.map(strings, (string) => { + const expected = caseName === 'start' && string === 'FOO BAR' ? string : converted; + return func(func(string)) === expected; + }); + expect(actual).toEqual(_.map(strings, stubTrue)); + }); + + it(`\`_.${methodName}\` should remove contraction apostrophes`, () => { + const postfixes = ['d', 'll', 'm', 're', 's', 't', 've']; + + _.each(["'", '\u2019'], (apos) => { + const actual = _.map(postfixes, (postfix) => + func(`a b${apos}${postfix} c`), + ); + const expected = _.map(postfixes, (postfix) => { + switch (caseName) { + case 'snakeCase': + return `a_b${postfix}_c`; + case 'snakeCaseWithNumbers': + return `a_b${postfix}_c`; + } + }); + expect(actual).toEqual(expected); + }); + }); + + it(`\`_.${methodName}\` should remove Latin mathematical operators`, () => { + const actual = _.map(['\xd7', '\xf7'], func); + expect(actual).toEqual(['', '']); + }); + + it(`\`_.${methodName}\` should coerce \`string\` to a string`, () => { + const string = 'foo bar'; + expect(func(Object(string))).toBe(converted); + expect(func({toString: _.constant(string)})).toBe(converted); + }); + + it(`\`_.${methodName}\` should return an empty string for empty values`, () => { + const values = [, null, undefined, '']; + const expected = _.map(values, stubString); + + const actual = _.map(values, (value, index) => + index ? func(value) : func(), + ); + + expect(actual).toEqual(expected); + }); + }); +}); \ No newline at end of file diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 83b24aee15..6bde2e9eb2 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -1,7 +1,5 @@ const _ = require("lodash"); -const util = require("util"); - const { input, output } = require(`./data/warehouse/events.js`); const { opInput, @@ -21,6 +19,7 @@ const { const { validTimestamp } = require("../../src/warehouse/util.js"); +const {transformTableName, transformColumnName} = require("../../src/warehouse/v1/util"); const {isBlank} = require("../../src/warehouse/config/helpers.js"); const version = "v0"; @@ -1010,28 +1009,6 @@ describe("Add receivedAt for events missing it", () => { }); describe("Integration options", () => { - describe("Destination config options", () => { - destConfig.scenarios().forEach(scenario => { - it(scenario.name, () => { - if (scenario.skipUsersTable !== null) { - scenario.event.destination.Config.skipUsersTable = scenario.skipUsersTable - } - if (scenario.skipTracksTable !== null) { - scenario.event.destination.Config.skipTracksTable = scenario.skipTracksTable - } - - transformers.forEach((transformer, index) => { - const received = transformer.process(scenario.event); - expect(received).toHaveLength(scenario.expected.length); - for (const i in received) { - const evt = received[i]; - expect(evt.data.id ? evt.data.id : evt.data.ID).toEqual(scenario.expected[i].id); - expect(evt.metadata.table.toLowerCase()).toEqual(scenario.expected[i].table); - } - }); - }); - }); - }); describe("track", () => { it("should generate two events for every track call", () => { const i = opInput("track"); @@ -1058,7 +1035,7 @@ describe("Integration options", () => { }); describe("json paths", () => { - const output = (config, provider) => { + const output = (eventType, config, provider) => { switch (provider) { case "rs": return _.cloneDeep(config.output.rs); @@ -1068,6 +1045,14 @@ describe("Integration options", () => { return _.cloneDeep(config.output.postgres); case "snowflake": return _.cloneDeep(config.output.snowflake); + case "s3_datalake": + case "gcs_datalake": + case "azure_datalake": + if (eventType === 'identifies') { + return _.cloneDeep(config.output.datalake); + } else { + return _.cloneDeep(config.output.default); + } default: return _.cloneDeep(config.output.default); } @@ -1103,14 +1088,14 @@ describe("Integration options", () => { const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(config, integrations[index])); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(config, integrations[index])); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) }); } @@ -1235,6 +1220,306 @@ describe("isBlank", () => { } }); +describe("Destination config", () => { + describe("skipUsersTable, skipTracksTable", () => { + destConfig.scenarios().forEach(scenario => { + it(scenario.name, () => { + if (scenario.skipUsersTable !== null) { + scenario.event.destination.Config.skipUsersTable = scenario.skipUsersTable + } + if (scenario.skipTracksTable !== null) { + scenario.event.destination.Config.skipTracksTable = scenario.skipTracksTable + } + + transformers.forEach((transformer, index) => { + const received = transformer.process(scenario.event); + expect(received).toHaveLength(scenario.expected.length); + for (const i in received) { + const evt = received[i]; + expect(evt.data.id ? evt.data.id : evt.data.ID).toEqual(scenario.expected[i].id); + expect(evt.metadata.table.toLowerCase()).toEqual(scenario.expected[i].table); + } + }); + }); + }); + }); + + describe('allowUsersContextTraits, underscoreDivideNumbers', () => { + describe("old destinations", () => { + it('with allowUsersContextTraits', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: { + allowUsersContextTraits: true + } + }, + message: { + context: { + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + }, + traits: { + lastname: "Mouse" + }, + type: "identify", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + const events = [output[0], output[1]]; // identifies and users event + const traitsToCheck = { + 'city': 'Disney', + 'country': 'USA', + 'email': 'mickey@disney.com', + 'firstname': 'Mickey' + }; + events.forEach(event => { + Object.entries(traitsToCheck).forEach(([trait, value]) => { + expect(event.data[integrationCasedString(integrations[index], trait)]).toEqual(value); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + }); + }); + }); + }); + + it('with underscoreDivideNumbers', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: { + underscoreDivideNumbers: true + }, + }, + message: { + context: { + 'attribute v3': 'some-value' + }, + event: "button clicked v2", + type: "track", + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v_2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); + }); + }); + }); + describe("new destinations", () => { + it('without allowUsersContextTraits', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: {} + }, + message: { + context: { + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + }, + traits: { + lastname: "Mouse" + }, + type: "identify", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const received = transformer.process(event); + const events = [received[0], received[1]]; // identifies and users event + const traitsToCheck = { + 'city': 'Disney', + 'country': 'USA', + 'email': 'mickey@disney.com', + 'firstname': 'Mickey' + }; + events.forEach(event => { + Object.entries(traitsToCheck).forEach(([trait, value]) => { + expect(event.data).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + }); + }); + }); + }); + + it('without underscoreDivideNumbers', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: {}, + }, + message: { + context: { + 'attribute v3': 'some-value' + }, + event: "button clicked v2", + type: "track", + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); + }); + }); + }); + }); +}); + +describe("validTimestamp", () => { + const testCases = [ + { + name: "undefined input should return false", + input: undefined, + expected: false, + }, + { + name: "negative year and time input should return false #1", + input: '-0001-11-30T00:00:00+0000', + expected: false, + }, + { + name: "negative year and time input should return false #2", + input: '-2023-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "negative year and time input should return false #3", + input: '-1900-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "positive year and time input should return false", + input: '+2023-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "valid timestamp input should return true", + input: '2023-06-14T05:23:59.244Z', + expected: true, + }, + { + name: "non-date string input should return false", + input: 'abc', + expected: false, + }, + { + name: "malicious string input should return false", + input: '%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216Windows%u2216win%u002ein', + expected: false, + }, + { + name: "empty string input should return false", + input: '', + expected: false, + }, + { + name: "valid date input should return true", + input: '2023-06-14', + expected: true, + }, + { + name: "time-only input should return false", + input: '05:23:59.244Z', + expected: false, + }, + { + name: "non-string input should return false", + input: {abc: 123}, + expected: false, + }, + { + name: "object with toString method input should return false", + input: { + toString: '2023-06-14T05:23:59.244Z' + }, + expected: false, + }, + ]; + for (const testCase of testCases) { + it(`should return ${testCase.expected} for ${testCase.name}`, () => { + expect(validTimestamp(testCase.input)).toEqual(testCase.expected); + }); + } +}); + +describe("isBlank", () => { + const testCases = [ + { + name: "null", + input: null, + expected: true + }, + { + name: "empty string", + input: "", + expected: true + }, + { + name: "non-empty string", + input: "test", + expected: false + }, + { + name: "numeric value", + input: 1634762544, + expected: false + }, + { + name: "object with toString property", + input: { + toString: '2023-06-14T05:23:59.244Z' + }, + expected: false + }, + ]; + for (const testCase of testCases) { + it(`should return ${testCase.expected} for ${testCase.name}`, () => { + expect(isBlank(testCase.input)).toEqual(testCase.expected); + }); + } +}); + describe("context traits", () => { const testCases = [ { @@ -1421,3 +1706,651 @@ describe("group traits", () => { }); } }) + +describe("transformColumnName", () => { + describe('with Blendo Casing', () => { + const testCases = [ + { + description: 'should convert special characters other than "\\" or "$" to underscores', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'column@Name$1', + expected: 'column_name$1', + }, + { + description: 'should add underscore if name does not start with an alphabet or underscore', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: '1CComega', + expected: '_1ccomega', + }, + { + description: 'should handle non-ASCII characters by converting to underscores', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'Cízǔ', + expected: 'c_z_', + }, + { + description: 'should transform CamelCase123Key to camelcase123key', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'CamelCase123Key', + expected: 'camelcase123key', + }, + { + description: 'should preserve "\\" and "$" characters', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'path to $1,00,000', + expected: 'path_to_$1_00_000', + }, + { + description: 'should handle a mix of characters, numbers, and special characters', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'CamelCase123Key_with$special\\chars', + expected: 'camelcase123key_with$special\\chars', + }, + { + description: 'should limit length to 63 characters for postgres provider', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'a'.repeat(70), + expected: 'a'.repeat(63), + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformColumnName(options, input); + expect(result).toBe(expected); + }); + }); + }); + + describe('without Blendo Casing (underscoreDivideNumbers=true)', () => { + const testCases = [ + { + description: 'should remove symbols and join continuous letters and numbers with a single underscore', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '&4yasdfa(84224_fs9##_____*3q', + expected: '_4_yasdfa_84224_fs_9_3_q', + }, + { + description: 'should transform "omega" to "omega"', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega', + expected: 'omega', + }, + { + description: 'should transform "omega v2" to "omega_v_2"', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega v2', + expected: 'omega_v_2', + }, + { + description: 'should prepend underscore if name starts with a number', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '9mega', + expected: '_9_mega', + }, + { + description: 'should remove trailing special characters', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'mega&', + expected: 'mega', + }, + { + description: 'should replace special character in the middle with underscore', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'ome$ga', + expected: 'ome_ga', + }, + { + description: 'should not remove trailing $ character', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega$', + expected: 'omega', + }, + { + description: 'should handle spaces and special characters by converting to underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'ome_ ga', + expected: 'ome_ga', + }, + { + description: 'should handle multiple underscores and hyphens by reducing to single underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '9mega________-________90', + expected: '_9_mega_90', + }, + { + description: 'should handle non-ASCII characters by converting them to underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'Cízǔ', + expected: 'c_z', + }, + { + description: 'should transform CamelCase123Key to camel_case_123_key', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'CamelCase123Key', + expected: 'camel_case_123_key', + }, + { + description: 'should handle numbers and commas in the input', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'path to $1,00,000', + expected: 'path_to_1_00_000', + }, + { + description: 'should return an empty string if input contains no valid characters', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '@#$%', + expected: '', + }, + { + description: 'should keep underscores between letters and numbers', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'test123', + expected: 'test_123', + }, + { + description: 'should keep multiple underscore-number sequences', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'abc123def456', + expected: 'abc_123_def_456', + }, + { + description: 'should keep multiple underscore-number sequences', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'abc_123_def_456', + expected: 'abc_123_def_456', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformColumnName(options, input); + expect(result).toBe(expected); + }); + }); + }); + + describe('without Blendo Casing (underscoreDivideNumbers=false)', () => { + const testCases = [ + { + description: 'should remove symbols and join continuous letters and numbers with a single underscore', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: '&4yasdfa(84224_fs9##_____*3q', + expected: '_4yasdfa_84224_fs9_3q', + }, + { + description: 'should transform "omega" to "omega"', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'omega', + expected: 'omega', + }, + { + description: 'should transform "omega v2" to "omega_v_2"', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'omega v2', + expected: 'omega_v2', + }, + { + description: 'should prepend underscore if name starts with a number', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: '9mega', + expected: '_9mega', + }, + { + description: 'should remove trailing special characters', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'mega&', + expected: 'mega', + }, + { + description: 'should replace special character in the middle with underscore', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'ome$ga', + expected: 'ome_ga', + }, + { + description: 'should not remove trailing $ character', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega$', + expected: 'omega', + }, + { + description: 'should handle spaces and special characters by converting to underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'ome_ ga', + expected: 'ome_ga', + }, + { + description: 'should handle multiple underscores and hyphens by reducing to single underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: '9mega________-________90', + expected: '_9mega_90', + }, + { + description: 'should handle non-ASCII characters by converting them to underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'Cízǔ', + expected: 'c_z', + }, + { + description: 'should transform CamelCase123Key to camel_case_123_key', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'CamelCase123Key', + expected: 'camel_case123_key', + }, + { + description: 'should handle numbers and commas in the input', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'path to $1,00,000', + expected: 'path_to_1_00_000', + }, + { + description: 'should return an empty string if input contains no valid characters', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: '@#$%', + expected: '', + }, + { + description: 'should keep underscores between letters and numbers', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'test123', + expected: 'test123', + }, + { + description: 'should keep multiple underscore-number sequences', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'abc123def456', + expected: 'abc123_def456', + }, + { + description: 'should keep multiple underscore-number sequences', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'abc_123_def_456', + expected: 'abc_123_def_456', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformColumnName(options, input); + expect(result).toBe(expected); + }); + }); + }); +}) + +describe("transformTableName", () => { + describe('with Blendo Casing', () => { + const testCases = [ + { + description: 'should convert name to Blendo casing (lowercase) when Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'TableName123', + expected: 'tablename123', + }, + { + description: 'should trim spaces and convert to Blendo casing (lowercase) when Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: ' TableName ', + expected: 'tablename', + }, + { + description: 'should return an empty string when input is empty and Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: '', + expected: '', + }, + { + description: 'should handle names with special characters and convert to Blendo casing (lowercase)', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'Table@Name!', + expected: 'table@name!', + }, + { + description: 'should convert a mixed-case name to Blendo casing (lowercase) when Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'CaMeLcAsE', + expected: 'camelcase', + }, + { + description: 'should keep an already lowercase name unchanged with Blendo casing enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'lowercase', + expected: 'lowercase', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformTableName(options, input); + expect(result).toBe(expected); + }); + }); + }); + + describe('without Blendo Casing (underscoreDivideNumbers=true)', () => { + const testCases = [ + { + description: 'should remove symbols and join continuous letters and numbers with a single underscore', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '&4yasdfa(84224_fs9##_____*3q', + expected: '_4_yasdfa_84224_fs_9_3_q', + }, + { + description: 'should transform "omega" to "omega"', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega', + expected: 'omega', + }, + { + description: 'should transform "omega v2" to "omega_v_2"', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega v2', + expected: 'omega_v_2', + }, + { + description: 'should prepend underscore if name starts with a number', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '9mega', + expected: '_9_mega', + }, + { + description: 'should remove trailing special characters', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'mega&', + expected: 'mega', + }, + { + description: 'should replace special character in the middle with underscore', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'ome$ga', + expected: 'ome_ga', + }, + { + description: 'should not remove trailing $ character', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega$', + expected: 'omega', + }, + { + description: 'should handle spaces and special characters by converting to underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'ome_ ga', + expected: 'ome_ga', + }, + { + description: 'should handle multiple underscores and hyphens by reducing to single underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '9mega________-________90', + expected: '_9_mega_90', + }, + { + description: 'should handle non-ASCII characters by converting them to underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'Cízǔ', + expected: 'c_z', + }, + { + description: 'should transform CamelCase123Key to camel_case_123_key', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'CamelCase123Key', + expected: 'camel_case_123_key', + }, + { + description: 'should handle numbers and commas in the input', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'path to $1,00,000', + expected: 'path_to_1_00_000', + }, + { + description: 'should return an empty string if input contains no valid characters', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '@#$%', + expected: '', + }, + { + description: 'should keep underscores between letters and numbers', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'test123', + expected: 'test_123', + }, + { + description: 'should keep multiple underscore-number sequences', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'abc123def456', + expected: 'abc_123_def_456', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformTableName(options, input); + expect(result).toBe(expected); + }); + }); + }); + + describe('without Blendo Casing (underscoreDivideNumbers=false)', () => { + const testCases = [ + { + description: 'should remove symbols and join continuous letters and numbers with a single underscore', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: '&4yasdfa(84224_fs9##_____*3q', + expected: '_4yasdfa_84224_fs9_3q', + }, + { + description: 'should transform "omega" to "omega"', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'omega', + expected: 'omega', + }, + { + description: 'should transform "omega v2" to "omega_v_2"', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'omega v2', + expected: 'omega_v2', + }, + { + description: 'should prepend underscore if name starts with a number', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: '9mega', + expected: '_9mega', + }, + { + description: 'should remove trailing special characters', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'mega&', + expected: 'mega', + }, + { + description: 'should replace special character in the middle with underscore', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'ome$ga', + expected: 'ome_ga', + }, + { + description: 'should not remove trailing $ character', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega$', + expected: 'omega', + }, + { + description: 'should handle spaces and special characters by converting to underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'ome_ ga', + expected: 'ome_ga', + }, + { + description: 'should handle multiple underscores and hyphens by reducing to single underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: '9mega________-________90', + expected: '_9mega_90', + }, + { + description: 'should handle non-ASCII characters by converting them to underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'Cízǔ', + expected: 'c_z', + }, + { + description: 'should transform CamelCase123Key to camel_case_123_key', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'CamelCase123Key', + expected: 'camel_case123_key', + }, + { + description: 'should handle numbers and commas in the input', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'path to $1,00,000', + expected: 'path_to_1_00_000', + }, + { + description: 'should return an empty string if input contains no valid characters', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: '@#$%', + expected: '', + }, + { + description: 'should keep underscores between letters and numbers', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'test123', + expected: 'test123', + }, + { + description: 'should keep multiple underscore-number sequences', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'abc123def456', + expected: 'abc123_def456', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformTableName(options, input); + expect(result).toBe(expected); + }); + }); + }); +}); \ No newline at end of file diff --git a/test/integrations/destinations/gcs_datalake/processor/data.ts b/test/integrations/destinations/gcs_datalake/processor/data.ts index 46c7788709..26f438758d 100644 --- a/test/integrations/destinations/gcs_datalake/processor/data.ts +++ b/test/integrations/destinations/gcs_datalake/processor/data.ts @@ -62,6 +62,8 @@ export const data = [ syncFrequency: '30', tableSuffix: '', timeWindowLayout: '2006/01/02/15', + allowUsersContextTraits: true, + underscoreDivideNumbers: true, }, Enabled: true, }, diff --git a/test/integrations/destinations/intercom/network.ts b/test/integrations/destinations/intercom/network.ts index 2f90beac40..0a86ce3c89 100644 --- a/test/integrations/destinations/intercom/network.ts +++ b/test/integrations/destinations/intercom/network.ts @@ -1041,5 +1041,42 @@ const deliveryCallsData = [ }, }, }, + { + httpReq: { + method: 'post', + url: 'https://api.intercom.io/contacts/search', + data: { + query: { + operator: 'AND', + value: [{ field: 'external_id', operator: '=', value: '10156' }], + }, + }, + headers: { ...commonHeaders, 'Intercom-Version': '2.10', 'User-Agent': 'RudderStack' }, + }, + httpRes: { + status: 200, + statusText: 'ok', + data: { + type: 'list', + total_count: 1, + pages: { + type: 'pages', + page: 1, + per_page: 50, + total_pages: 1, + }, + data: [ + { + type: 'contact', + id: '7070129940741e45d040', + workspace_id: 'rudderWorkspace', + external_id: 'user@2', + role: 'user', + email: 'test+2@rudderlabs.com', + }, + ], + }, + }, + }, ]; export const networkCallsData = [...deleteNwData, ...deliveryCallsData]; diff --git a/test/integrations/destinations/intercom/processor/identifyTestData.ts b/test/integrations/destinations/intercom/processor/identifyTestData.ts index 49f3a400d1..f078536b30 100644 --- a/test/integrations/destinations/intercom/processor/identifyTestData.ts +++ b/test/integrations/destinations/intercom/processor/identifyTestData.ts @@ -80,6 +80,10 @@ const user3Traits = { name: 'Test Rudderlabs', phone: '+91 9999999999', email: 'test@rudderlabs.com', + custom_attributes: { + ca1: 'value1', + ca2: 'value2', + }, }; const user4Traits = { @@ -170,6 +174,10 @@ const expectedUser3Traits = { name: 'Test Rudderlabs', phone: '+91 9999999999', email: 'test@rudderlabs.com', + custom_attributes: { + ca1: 'value1', + ca2: 'value2', + }, }; const expectedUser4Traits = { @@ -233,6 +241,17 @@ const expectedUser6Traits = { ], }; +const expectedUser7Traits = { + custom_attributes: { + anonymousId: '58b21c2d-f8d5-4410-a2d0-b268a26b7e33', + key1: 'value1', + }, + email: 'test_1@test.com', + name: 'Test Name', + phone: '9876543210', + signed_up_at: 1601493060, +}; + const timestamp = '2023-11-22T10:12:44.757+05:30'; const originalTimestamp = '2023-11-10T14:42:44.724Z'; @@ -1024,4 +1043,63 @@ export const identifyTestData = [ }, }, }, + { + id: 'intercom-identify-test-16', + name: 'intercom', + description: 'V1 version : Identify test with different lookup field than email', + scenario: 'Business', + successCriteria: + 'Response status code should be 200 and response should contain update user payload with all traits', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: v2Destination, + message: { + context: { + externalId: [ + { + id: '10156', + type: 'INTERCOM-customer', + identifierType: 'user_id', + }, + ], + traits: { ...user5Traits, external_id: '10156' }, + }, + type: 'identify', + timestamp, + originalTimestamp, + integrations: { + INTERCOM: { + lookup: 'external_id', + }, + }, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + endpoint: `${v2Endpoint}/7070129940741e45d040`, + headers: v2Headers, + method: 'PUT', + JSON: expectedUser7Traits, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ];