From 3363156260a6a1c72055db458caacc3b5fd7deee Mon Sep 17 00:00:00 2001 From: achettyiitr Date: Mon, 23 Sep 2024 19:45:34 +0530 Subject: [PATCH] fix: allow users context traits and underscore divide numbers configuration --- src/warehouse/index.js | 34 +- src/warehouse/snakecase/snakecase.js | 37 + src/warehouse/snakecase/unicodeWords.js | 94 ++ src/warehouse/util.js | 11 +- src/warehouse/v0/util.js | 87 -- src/warehouse/v1/util.js | 16 +- test/__tests__/data/warehouse/events.js | 28 +- .../warehouse/integration_options_events.js | 9 +- .../integrations/jsonpaths/legacy/aliases.js | 5 +- .../integrations/jsonpaths/legacy/extract.js | 5 +- .../integrations/jsonpaths/legacy/groups.js | 5 +- .../jsonpaths/legacy/identifies.js | 5 +- .../integrations/jsonpaths/legacy/pages.js | 5 +- .../integrations/jsonpaths/legacy/screens.js | 5 +- .../integrations/jsonpaths/legacy/tracks.js | 5 +- .../integrations/jsonpaths/new/aliases.js | 5 +- .../integrations/jsonpaths/new/extract.js | 5 +- .../integrations/jsonpaths/new/groups.js | 5 +- .../integrations/jsonpaths/new/identifies.js | 5 +- .../integrations/jsonpaths/new/pages.js | 5 +- .../integrations/jsonpaths/new/screens.js | 5 +- .../integrations/jsonpaths/new/tracks.js | 5 +- test/__tests__/snakecase.test.js | 497 +++++++++ test/__tests__/warehouse.test.js | 973 +++++++++++++++++- .../gcs_datalake/processor/data.ts | 2 + 25 files changed, 1697 insertions(+), 161 deletions(-) create mode 100644 src/warehouse/snakecase/snakecase.js create mode 100644 src/warehouse/snakecase/unicodeWords.js delete mode 100644 src/warehouse/v0/util.js create mode 100644 test/__tests__/snakecase.test.js diff --git a/src/warehouse/index.js b/src/warehouse/index.js index 3c2b04079d..54368da9fc 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -643,6 +643,18 @@ function processWarehouseMessage(message, options) { const skipReservedKeywordsEscaping = options.integrationOptions.skipReservedKeywordsEscaping || false; + // underscoreDivideNumbers when set to false, if a column has a format like "_v_3_", it will be formatted to "_v3_" + // underscoreDivideNumbers when set to true, if a column has a format like "_v_3_", we keep it like that + // For older destinations, it will come as true and for new destinations this config will not be present which means we will treat it as false. + options.underscoreDivideNumbers = options.destConfig?.underscoreDivideNumbers || false; + + // allowUsersContextTraits when set to true, if context.traits.* is present, it will be added as context_traits_* and *, + // e.g., for context.traits.name, context_traits_name and name will be added to the user's table. + // allowUsersContextTraits when set to false, if context.traits.* is present, it will be added only as context_traits_* + // e.g., for context.traits.name, only context_traits_name will be added to the user's table. + // For older destinations, it will come as true, and for new destinations this config will not be present, which means we will treat it as false. + const allowUsersContextTraits = options.destConfig?.allowUsersContextTraits || false; + addJsonKeysToOptions(options); if (isBlank(message.messageId)) { @@ -898,16 +910,18 @@ function processWarehouseMessage(message, options) { `${eventType + '_userProperties_'}`, 2, ); - setDataFromInputAndComputeColumnTypes( - utils, - eventType, - commonProps, - message.context ? message.context.traits : {}, - commonColumnTypes, - options, - `${eventType + '_context_traits_'}`, - 3, - ); + if (allowUsersContextTraits) { + setDataFromInputAndComputeColumnTypes( + utils, + eventType, + commonProps, + message.context ? message.context.traits : {}, + commonColumnTypes, + options, + `${eventType + '_context_traits_'}`, + 3, + ); + } setDataFromInputAndComputeColumnTypes( utils, eventType, diff --git a/src/warehouse/snakecase/snakecase.js b/src/warehouse/snakecase/snakecase.js new file mode 100644 index 0000000000..1e6586e7f2 --- /dev/null +++ b/src/warehouse/snakecase/snakecase.js @@ -0,0 +1,37 @@ +const { toString } = require('lodash'); +const { unicodeWords, unicodeWordsWithNumbers } = require('./unicodeWords'); + +const hasUnicodeWord = RegExp.prototype.test.bind( + /[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/, +); + +/** Used to match words composed of alphanumeric characters. */ +// eslint-disable-next-line no-control-regex +const reAsciiWord = /[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g; + +function asciiWords(string) { + return string.match(reAsciiWord); +} + +function words(string) { + const result = hasUnicodeWord(string) ? unicodeWords(string) : asciiWords(string); + return result || []; +} + +function wordsWithNumbers(string) { + const result = hasUnicodeWord(string) ? unicodeWordsWithNumbers(string) : asciiWords(string); + return result || []; +} + +const snakeCase = (string) => + words(toString(string).replace(/['\u2019]/g, '')).reduce( + (result, word, index) => result + (index ? '_' : '') + word.toLowerCase(), + '', + ); +const snakeCaseWithNumbers = (string) => + wordsWithNumbers(toString(string).replace(/['\u2019]/g, '')).reduce( + (result, word, index) => result + (index ? '_' : '') + word.toLowerCase(), + '', + ); + +module.exports = { words, wordsWithNumbers, snakeCase, snakeCaseWithNumbers }; diff --git a/src/warehouse/snakecase/unicodeWords.js b/src/warehouse/snakecase/unicodeWords.js new file mode 100644 index 0000000000..d7b15806c7 --- /dev/null +++ b/src/warehouse/snakecase/unicodeWords.js @@ -0,0 +1,94 @@ +/** Used to compose unicode character classes. */ +const rsAstralRange = '\\ud800-\\udfff'; +const rsComboMarksRange = '\\u0300-\\u036f'; +const reComboHalfMarksRange = '\\ufe20-\\ufe2f'; +const rsComboSymbolsRange = '\\u20d0-\\u20ff'; +const rsComboMarksExtendedRange = '\\u1ab0-\\u1aff'; +const rsComboMarksSupplementRange = '\\u1dc0-\\u1dff'; +const rsComboRange = + rsComboMarksRange + + reComboHalfMarksRange + + rsComboSymbolsRange + + rsComboMarksExtendedRange + + rsComboMarksSupplementRange; +const rsDingbatRange = '\\u2700-\\u27bf'; +const rsLowerRange = 'a-z\\xdf-\\xf6\\xf8-\\xff'; +const rsMathOpRange = '\\xac\\xb1\\xd7\\xf7'; +const rsNonCharRange = '\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf'; +const rsPunctuationRange = '\\u2000-\\u206f'; +const rsSpaceRange = + ' \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000'; +const rsUpperRange = 'A-Z\\xc0-\\xd6\\xd8-\\xde'; +const rsVarRange = '\\ufe0e\\ufe0f'; +const rsBreakRange = rsMathOpRange + rsNonCharRange + rsPunctuationRange + rsSpaceRange; + +/** Used to compose unicode capture groups. */ +const rsApos = "['\u2019]"; +const rsBreak = `[${rsBreakRange}]`; +const rsCombo = `[${rsComboRange}]`; +const rsDigit = '\\d'; +const rsDingbat = `[${rsDingbatRange}]`; +const rsLower = `[${rsLowerRange}]`; +const rsMisc = `[^${rsAstralRange}${rsBreakRange + rsDigit + rsDingbatRange + rsLowerRange + rsUpperRange}]`; +const rsFitz = '\\ud83c[\\udffb-\\udfff]'; +const rsModifier = `(?:${rsCombo}|${rsFitz})`; +const rsNonAstral = `[^${rsAstralRange}]`; +const rsRegional = '(?:\\ud83c[\\udde6-\\uddff]){2}'; +const rsSurrPair = '[\\ud800-\\udbff][\\udc00-\\udfff]'; +const rsUpper = `[${rsUpperRange}]`; +const rsZWJ = '\\u200d'; + +/** Used to compose unicode regexes. */ +const rsMiscLower = `(?:${rsLower}|${rsMisc})`; +const rsMiscUpper = `(?:${rsUpper}|${rsMisc})`; +const rsOptContrLower = `(?:${rsApos}(?:d|ll|m|re|s|t|ve))?`; +const rsOptContrUpper = `(?:${rsApos}(?:D|LL|M|RE|S|T|VE))?`; +const reOptMod = `${rsModifier}?`; +const rsOptVar = `[${rsVarRange}]?`; +const rsOptJoin = `(?:${rsZWJ}(?:${[rsNonAstral, rsRegional, rsSurrPair].join('|')})${rsOptVar + reOptMod})*`; +const rsOrdLower = '\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])'; +const rsOrdUpper = '\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])'; +const rsSeq = rsOptVar + reOptMod + rsOptJoin; +const rsEmoji = `(?:${[rsDingbat, rsRegional, rsSurrPair].join('|')})${rsSeq}`; + +const reUnicodeWords = RegExp( + [ + `${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`, // Regular words, lowercase letters followed by optional contractions + `${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`, // Miscellaneous uppercase characters with optional contractions + `${rsUpper}?${rsMiscLower}+${rsOptContrLower}`, // Miscellaneous lowercase sequences with optional contractions + `${rsUpper}+${rsOptContrUpper}`, // All uppercase words with optional contractions (e.g., "THIS") + rsOrdUpper, // Ordinals for uppercase (e.g., "1ST", "2ND") + rsOrdLower, // Ordinals for lowercase (e.g., "1st", "2nd") + `${rsDigit}+`, // Pure digits (e.g., "123") + rsEmoji, // Emojis (e.g., 😀, ❤️) + ].join('|'), + 'g', +); + +const reUnicodeWordsWithNumbers = RegExp( + [ + `${rsUpper}?${rsLower}+${rsDigit}+`, // Lowercase letters followed by digits (e.g., "abc123") + `${rsUpper}+${rsDigit}+`, // Uppercase letters followed by digits (e.g., "ABC123") + `${rsDigit}+${rsUpper}?${rsLower}+`, // Digits followed by lowercase letters (e.g., "123abc") + `${rsDigit}+${rsUpper}+`, // Digits followed by uppercase letters (e.g., "123ABC") + `${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`, // Regular words, lowercase letters followed by optional contractions + `${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`, // Miscellaneous uppercase characters with optional contractions + `${rsUpper}?${rsMiscLower}+${rsOptContrLower}`, // Miscellaneous lowercase sequences with optional contractions + `${rsUpper}+${rsOptContrUpper}`, // All uppercase words with optional contractions (e.g., "THIS") + rsOrdUpper, // Ordinals for uppercase (e.g., "1ST", "2ND") + rsOrdLower, // Ordinals for lowercase (e.g., "1st", "2nd") + `${rsDigit}+`, // Pure digits (e.g., "123") + rsEmoji, // Emojis (e.g., 😀, ❤️) + ].join('|'), + 'g', +); + +function unicodeWords(string) { + return string.match(reUnicodeWords); +} + +function unicodeWordsWithNumbers(string) { + return string.match(reUnicodeWordsWithNumbers); +} + +module.exports = { unicodeWords, unicodeWordsWithNumbers }; diff --git a/src/warehouse/util.js b/src/warehouse/util.js index b4b22721fd..7f4e224a34 100644 --- a/src/warehouse/util.js +++ b/src/warehouse/util.js @@ -1,7 +1,5 @@ -const _ = require('lodash'); const get = require('get-value'); -const v0 = require('./v0/util'); const v1 = require('./v1/util'); const { PlatformError, InstrumentationError } = require('@rudderstack/integrations-lib'); const { isBlank } = require('./config/helpers'); @@ -112,14 +110,7 @@ function validTimestamp(input) { } function getVersionedUtils(schemaVersion) { - switch (schemaVersion) { - case 'v0': - return v0; - case 'v1': - return v1; - default: - return v1; - } + return v1; } function isRudderSourcesEvent(event) { diff --git a/src/warehouse/v0/util.js b/src/warehouse/v0/util.js deleted file mode 100644 index 5917f8ea08..0000000000 --- a/src/warehouse/v0/util.js +++ /dev/null @@ -1,87 +0,0 @@ -const reservedANSIKeywordsMap = require('../config/ReservedKeywords.json'); -const { isDataLakeProvider } = require('../config/helpers'); - -const toSnakeCase = (str) => { - if (!str) { - return ''; - } - return String(str) - .replace(/^[^A-Za-z0-9]*|[^A-Za-z0-9]*$/g, '') - .replace(/([a-z])([A-Z])/g, (m, a, b) => `${a}_${b.toLowerCase()}`) - .replace(/[^A-Za-z0-9]+|_+/g, '_') - .toLowerCase(); -}; - -function toSafeDBString(provider, name = '') { - let parsedStr = name; - if (parseInt(name[0], 10) >= 0) { - parsedStr = `_${name}`; - } - parsedStr = parsedStr.replace(/[^a-zA-Z0-9_]+/g, ''); - if (isDataLakeProvider(provider)) { - return parsedStr; - } - switch (provider) { - case 'postgres': - return parsedStr.substr(0, 63); - default: - return parsedStr.substr(0, 127); - } -} - -function safeTableName(provider, name = '') { - let tableName = name; - if (tableName === '') { - tableName = 'STRINGEMPTY'; - } - if (provider === 'snowflake') { - tableName = tableName.toUpperCase(); - } - if (provider === 'rs' || isDataLakeProvider(provider)) { - tableName = tableName.toLowerCase(); - } - if (provider === 'postgres') { - tableName = tableName.substr(0, 63); - tableName = tableName.toLowerCase(); - } - if (reservedANSIKeywordsMap[provider.toUpperCase()][tableName.toUpperCase()]) { - tableName = `_${tableName}`; - } - return tableName; -} - -function safeColumnName(provider, name = '') { - let columnName = name; - if (columnName === '') { - columnName = 'STRINGEMPTY'; - } - if (provider === 'snowflake') { - columnName = columnName.toUpperCase(); - } - if (provider === 'rs' || isDataLakeProvider(provider)) { - columnName = columnName.toLowerCase(); - } - if (provider === 'postgres') { - columnName = columnName.substr(0, 63); - columnName = columnName.toLowerCase(); - } - if (reservedANSIKeywordsMap[provider.toUpperCase()][columnName.toUpperCase()]) { - columnName = `_${columnName}`; - } - return columnName; -} - -function transformTableName(name = '') { - return toSnakeCase(name); -} - -function transformColumnName(provider, name = '') { - return toSafeDBString(provider, name); -} - -module.exports = { - safeColumnName, - safeTableName, - transformColumnName, - transformTableName, -}; diff --git a/src/warehouse/v1/util.js b/src/warehouse/v1/util.js index 1c44a2385e..d1289bc674 100644 --- a/src/warehouse/v1/util.js +++ b/src/warehouse/v1/util.js @@ -1,8 +1,7 @@ -const _ = require('lodash'); - const reservedANSIKeywordsMap = require('../config/ReservedKeywords.json'); const { isDataLakeProvider } = require('../config/helpers'); const { TransformationError } = require('@rudderstack/integrations-lib'); +const { snakeCase, snakeCaseWithNumbers } = require('../snakecase/snakecase'); function safeTableName(options, name = '') { const { provider } = options; @@ -82,7 +81,7 @@ function safeColumnName(options, name = '') { path to $1,00,000 to path_to_1_00_000 return an empty string if it couldn't find a char if its ascii value doesnt belong to numbers or english alphabets */ -function transformName(provider, name = '') { +function transformName(options, provider, name = '') { const extractedValues = []; let extractedValue = ''; for (let i = 0; i < name.length; i += 1) { @@ -104,14 +103,17 @@ function transformName(provider, name = '') { if (extractedValue !== '') { extractedValues.push(extractedValue); } + const underscoreDivideNumbers = options?.underscoreDivideNumbers || false; + const snakeCaseFn = underscoreDivideNumbers ? snakeCase : snakeCaseWithNumbers; + let key = extractedValues.join('_'); if (name.startsWith('_')) { // do not remove leading underscores to allow esacaping rudder keywords with underscore // _timestamp -> _timestamp // __timestamp -> __timestamp - key = name.match(/^_*/)[0] + _.snakeCase(key.replace(/^_*/, '')); + key = name.match(/^_*/)[0] + snakeCaseFn(key.replace(/^_*/, '')); } else { - key = _.snakeCase(key); + key = snakeCaseFn(key); } if (key !== '' && key.charCodeAt(0) >= 48 && key.charCodeAt(0) <= 57) { @@ -150,7 +152,7 @@ function toBlendoCase(name = '') { function transformTableName(options, name = '') { const useBlendoCasing = options.integrationOptions?.useBlendoCasing || false; - return useBlendoCasing ? toBlendoCase(name) : transformName('', name); + return useBlendoCasing ? toBlendoCase(name) : transformName(options, '', name); } function transformColumnName(options, name = '') { @@ -158,7 +160,7 @@ function transformColumnName(options, name = '') { const useBlendoCasing = options.integrationOptions?.useBlendoCasing || false; return useBlendoCasing ? transformNameToBlendoCase(provider, name) - : transformName(provider, name); + : transformName(options, provider, name); } module.exports = { diff --git a/test/__tests__/data/warehouse/events.js b/test/__tests__/data/warehouse/events.js index ef9cc21096..bca6f776be 100644 --- a/test/__tests__/data/warehouse/events.js +++ b/test/__tests__/data/warehouse/events.js @@ -8,7 +8,9 @@ const sampleEvents = { Config: { restApiKey: "dummyApiKey", prefixProperties: true, - useNativeSDK: false + useNativeSDK: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, DestinationDefinition: { DisplayName: "Braze", @@ -514,7 +516,9 @@ const sampleEvents = { Config: { restApiKey: "dummyApiKey", prefixProperties: true, - useNativeSDK: false + useNativeSDK: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, DestinationDefinition: { DisplayName: "Braze", @@ -1026,7 +1030,9 @@ const sampleEvents = { Config: { restApiKey: "dummyApiKey", prefixProperties: true, - useNativeSDK: false + useNativeSDK: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, DestinationDefinition: { DisplayName: "Braze", @@ -1283,7 +1289,9 @@ const sampleEvents = { mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, - trackNamedPages: false + trackNamedPages: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, Enabled: true }, @@ -1532,7 +1540,9 @@ const sampleEvents = { mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, - trackNamedPages: false + trackNamedPages: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, Enabled: true }, @@ -1752,7 +1762,9 @@ const sampleEvents = { Config: { apiKey: "dummyApiKey", prefixProperties: true, - useNativeSDK: false + useNativeSDK: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, DestinationDefinition: { DisplayName: "Kiss Metrics", @@ -2016,7 +2028,9 @@ const sampleEvents = { Config: { restApiKey: "dummyApiKey", prefixProperties: true, - useNativeSDK: false + useNativeSDK: false, + allowUsersContextTraits: true, + underscoreDivideNumbers: true }, DestinationDefinition: { DisplayName: "Braze", diff --git a/test/__tests__/data/warehouse/integration_options_events.js b/test/__tests__/data/warehouse/integration_options_events.js index 05a2d51abd..bb068d4857 100644 --- a/test/__tests__/data/warehouse/integration_options_events.js +++ b/test/__tests__/data/warehouse/integration_options_events.js @@ -20,7 +20,9 @@ const sampleEvents = { input: { destination: { Config: { - jsonPaths: " testMap.nestedMap, testArray" + jsonPaths: " testMap.nestedMap, testArray", + allowUsersContextTraits: true, + underscoreDivideNumbers: true } }, message: { @@ -731,7 +733,10 @@ const sampleEvents = { users: { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { type: "identify", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js index ee748a1a2b..30cce51fb7 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js index 906c3ada23..9d38ecc292 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js index 13b243f3a7..bbae993b27 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js index 47b7f1209a..14b01d4cda 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { type: "identify", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js index 0aac8a3c23..dc2d3a3318 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js index bad325c908..9b789d2747 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js index 5070284e99..691ce57ca1 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js index 91f19c5c77..d5d57f85b9 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js index 7a36e4787e..a950b7f526 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js index f84f9d33ed..6979aa1100 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js index 3d6164b430..884770864a 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { type: "identify", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js index 136f355b21..d5282c1cfd 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js index b11b311ebb..3eec47b89d 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js index 7ed95685ff..6b411835db 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js @@ -1,7 +1,10 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/snakecase.test.js b/test/__tests__/snakecase.test.js new file mode 100644 index 0000000000..146e2e53c6 --- /dev/null +++ b/test/__tests__/snakecase.test.js @@ -0,0 +1,497 @@ +const _ = require('lodash'); +const {words, wordsWithNumbers, snakeCase, snakeCaseWithNumbers} = require("../../src/warehouse/snakecase/snakecase"); + +const burredLetters = [ + // Latin-1 Supplement letters. + '\xc0', + '\xc1', + '\xc2', + '\xc3', + '\xc4', + '\xc5', + '\xc6', + '\xc7', + '\xc8', + '\xc9', + '\xca', + '\xcb', + '\xcc', + '\xcd', + '\xce', + '\xcf', + '\xd0', + '\xd1', + '\xd2', + '\xd3', + '\xd4', + '\xd5', + '\xd6', + '\xd8', + '\xd9', + '\xda', + '\xdb', + '\xdc', + '\xdd', + '\xde', + '\xdf', + '\xe0', + '\xe1', + '\xe2', + '\xe3', + '\xe4', + '\xe5', + '\xe6', + '\xe7', + '\xe8', + '\xe9', + '\xea', + '\xeb', + '\xec', + '\xed', + '\xee', + '\xef', + '\xf0', + '\xf1', + '\xf2', + '\xf3', + '\xf4', + '\xf5', + '\xf6', + '\xf8', + '\xf9', + '\xfa', + '\xfb', + '\xfc', + '\xfd', + '\xfe', + '\xff', + // Latin Extended-A letters. + '\u0100', + '\u0101', + '\u0102', + '\u0103', + '\u0104', + '\u0105', + '\u0106', + '\u0107', + '\u0108', + '\u0109', + '\u010a', + '\u010b', + '\u010c', + '\u010d', + '\u010e', + '\u010f', + '\u0110', + '\u0111', + '\u0112', + '\u0113', + '\u0114', + '\u0115', + '\u0116', + '\u0117', + '\u0118', + '\u0119', + '\u011a', + '\u011b', + '\u011c', + '\u011d', + '\u011e', + '\u011f', + '\u0120', + '\u0121', + '\u0122', + '\u0123', + '\u0124', + '\u0125', + '\u0126', + '\u0127', + '\u0128', + '\u0129', + '\u012a', + '\u012b', + '\u012c', + '\u012d', + '\u012e', + '\u012f', + '\u0130', + '\u0131', + '\u0132', + '\u0133', + '\u0134', + '\u0135', + '\u0136', + '\u0137', + '\u0138', + '\u0139', + '\u013a', + '\u013b', + '\u013c', + '\u013d', + '\u013e', + '\u013f', + '\u0140', + '\u0141', + '\u0142', + '\u0143', + '\u0144', + '\u0145', + '\u0146', + '\u0147', + '\u0148', + '\u0149', + '\u014a', + '\u014b', + '\u014c', + '\u014d', + '\u014e', + '\u014f', + '\u0150', + '\u0151', + '\u0152', + '\u0153', + '\u0154', + '\u0155', + '\u0156', + '\u0157', + '\u0158', + '\u0159', + '\u015a', + '\u015b', + '\u015c', + '\u015d', + '\u015e', + '\u015f', + '\u0160', + '\u0161', + '\u0162', + '\u0163', + '\u0164', + '\u0165', + '\u0166', + '\u0167', + '\u0168', + '\u0169', + '\u016a', + '\u016b', + '\u016c', + '\u016d', + '\u016e', + '\u016f', + '\u0170', + '\u0171', + '\u0172', + '\u0173', + '\u0174', + '\u0175', + '\u0176', + '\u0177', + '\u0178', + '\u0179', + '\u017a', + '\u017b', + '\u017c', + '\u017d', + '\u017e', + '\u017f', +]; +const emojiVar = '\ufe0f'; +const flag = '\ud83c\uddfa\ud83c\uddf8'; +const heart = `\u2764${emojiVar}`; +const hearts = '\ud83d\udc95'; +const comboGlyph = `\ud83d\udc68\u200d${heart}\u200d\ud83d\udc8B\u200d\ud83d\udc68`; +const leafs = '\ud83c\udf42'; +const rocket = '\ud83d\ude80'; +const stubTrue = function () { + return true; +}; +const stubString = function () { + return ''; +}; + +describe('words', () => { + it('should match words containing Latin Unicode letters', () => { + const expected = _.map(burredLetters, (letter) => [letter]); + const actual = _.map(burredLetters, (letter) => words(letter)); + expect(actual).toEqual(expected); + }); + + it('should work with compound words', () => { + expect(words('12ft')).toEqual(['12', 'ft']); + expect(words('aeiouAreVowels')).toEqual(['aeiou', 'Are', 'Vowels']); + expect(words('enable 6h format')).toEqual(['enable', '6', 'h', 'format']); + expect(words('enable 24H format')).toEqual(['enable', '24', 'H', 'format']); + expect(words('isISO8601')).toEqual(['is', 'ISO', '8601']); + expect(words('LETTERSAeiouAreVowels')).toEqual(['LETTERS', 'Aeiou', 'Are', 'Vowels']); + expect(words('tooLegit2Quit')).toEqual(['too', 'Legit', '2', 'Quit']); + expect(words('walk500Miles')).toEqual(['walk', '500', 'Miles']); + expect(words('xhr2Request')).toEqual(['xhr', '2', 'Request']); + expect(words('XMLHttp')).toEqual(['XML', 'Http']); + expect(words('XmlHTTP')).toEqual(['Xml', 'HTTP']); + expect(words('XmlHttp')).toEqual(['Xml', 'Http']); + }); + + it('should work with compound words containing diacritical marks', () => { + expect(words('LETTERSÆiouAreVowels')).toEqual(['LETTERS', 'Æiou', 'Are', 'Vowels']); + expect(words('æiouAreVowels')).toEqual(['æiou', 'Are', 'Vowels']); + expect(words('æiou2Consonants')).toEqual(['æiou', '2', 'Consonants']); + }); + + it('should not treat contractions as separate words', () => { + const postfixes = ['d', 'll', 'm', 're', 's', 't', 've']; + + _.each(["'", '\u2019'], (apos) => { + _.times(2, (index) => { + const actual = _.map(postfixes, (postfix) => { + const string = `a b${apos}${postfix} c`; + return words(string[index ? 'toUpperCase' : 'toLowerCase']()); + }); + const expected = _.map(postfixes, (postfix) => { + const words = ['a', `b${apos}${postfix}`, 'c']; + return _.map(words, (word) => + word[index ? 'toUpperCase' : 'toLowerCase'](), + ); + }); + expect(actual).toEqual(expected); + }); + }); + }); + + it('should not treat ordinal numbers as separate words', () => { + const ordinals = ['1st', '2nd', '3rd', '4th']; + + _.times(2, (index) => { + const expected = _.map(ordinals, (ordinal) => [ + ordinal[index ? 'toUpperCase' : 'toLowerCase'](), + ]); + const actual = _.map(expected, (expectedWords) => words(expectedWords[0])); + expect(actual).toEqual(expected); + }); + }); + + it('should prevent ReDoS', () => { + const largeWordLen = 50000; + const largeWord = 'A'.repeat(largeWordLen); + const maxMs = 1000; + const startTime = _.now(); + + expect(words(`${largeWord}ÆiouAreVowels`)).toEqual([largeWord, 'Æiou', 'Are', 'Vowels']); + + const endTime = _.now(); + const timeSpent = endTime - startTime; + + expect(timeSpent).toBeLessThan(maxMs); + }); + + it('should account for astral symbols', () => { + const string = `A ${leafs}, ${comboGlyph}, and ${rocket}`; + expect(words(string)).toEqual(['A', leafs, comboGlyph, 'and', rocket]); + }); + + it('should account for regional symbols', () => { + const pair = flag.match(/\ud83c[\udde6-\uddff]/g); + const regionals = pair.join(' '); + + expect(words(flag)).toEqual([flag]); + expect(words(regionals)).toEqual([pair[0], pair[1]]); + }); + + it('should account for variation selectors', () => { + expect(words(heart)).toEqual([heart]); + }); + + it('should match lone surrogates', () => { + const pair = hearts.split(''); + const surrogates = `${pair[0]} ${pair[1]}`; + + expect(words(surrogates)).toEqual([]); + }); +}); + +describe('wordsWithoutNumbers', () => { + it('should match words containing Latin Unicode letters', () => { + const expected = _.map(burredLetters, (letter) => [letter]); + const actual = _.map(burredLetters, (letter) => wordsWithNumbers(letter)); + expect(actual).toEqual(expected); + }); + + it('should work with compound words', () => { + expect(wordsWithNumbers('12ft')).toEqual(['12ft']); + expect(wordsWithNumbers('aeiouAreVowels')).toEqual(['aeiou', 'Are', 'Vowels']); + expect(wordsWithNumbers('enable 6h format')).toEqual(['enable', '6h', 'format']); + expect(wordsWithNumbers('enable 24H format')).toEqual(['enable', '24H', 'format']); + expect(wordsWithNumbers('isISO8601')).toEqual(['is', 'ISO8601']); + expect(wordsWithNumbers('LETTERSAeiouAreVowels')).toEqual(['LETTERS', 'Aeiou', 'Are', 'Vowels']); + expect(wordsWithNumbers('tooLegit2Quit')).toEqual(['too', 'Legit2', 'Quit']); + expect(wordsWithNumbers('walk500Miles')).toEqual(['walk500', 'Miles']); + expect(wordsWithNumbers('xhr2Request')).toEqual(['xhr2', 'Request']); + expect(wordsWithNumbers('XMLHttp')).toEqual(['XML', 'Http']); + expect(wordsWithNumbers('XmlHTTP')).toEqual(['Xml', 'HTTP']); + expect(wordsWithNumbers('XmlHttp')).toEqual(['Xml', 'Http']); + }); + + it('should work with compound words containing diacritical marks', () => { + expect(wordsWithNumbers('LETTERSÆiouAreVowels')).toEqual(['LETTERS', 'Æiou', 'Are', 'Vowels']); + expect(wordsWithNumbers('æiouAreVowels')).toEqual(['æiou', 'Are', 'Vowels']); + expect(wordsWithNumbers('æiou2Consonants')).toEqual(['æiou2', 'Consonants']); + }); + + it('should not treat contractions as separate words', () => { + const postfixes = ['d', 'll', 'm', 're', 's', 't', 've']; + + _.each(["'", '\u2019'], (apos) => { + _.times(2, (index) => { + const actual = _.map(postfixes, (postfix) => { + const string = `a b${apos}${postfix} c`; + return wordsWithNumbers(string[index ? 'toUpperCase' : 'toLowerCase']()); + }); + const expected = _.map(postfixes, (postfix) => { + const words = ['a', `b${apos}${postfix}`, 'c']; + return _.map(words, (word) => + word[index ? 'toUpperCase' : 'toLowerCase'](), + ); + }); + expect(actual).toEqual(expected); + }); + }); + }); + + it('should not treat ordinal numbers as separate words', () => { + const ordinals = ['1st', '2nd', '3rd', '4th']; + + _.times(2, (index) => { + const expected = _.map(ordinals, (ordinal) => [ + ordinal[index ? 'toUpperCase' : 'toLowerCase'](), + ]); + const actual = _.map(expected, (expectedWords) => wordsWithNumbers(expectedWords[0])); + expect(actual).toEqual(expected); + }); + }); + + it('should prevent ReDoS', () => { + const largeWordLen = 50000; + const largeWord = 'A'.repeat(largeWordLen); + const maxMs = 1000; + const startTime = _.now(); + + expect(wordsWithNumbers(`${largeWord}ÆiouAreVowels`)).toEqual([largeWord, 'Æiou', 'Are', 'Vowels']); + + const endTime = _.now(); + const timeSpent = endTime - startTime; + + expect(timeSpent).toBeLessThan(maxMs); + }); + + it('should account for astral symbols', () => { + const string = `A ${leafs}, ${comboGlyph}, and ${rocket}`; + expect(wordsWithNumbers(string)).toEqual(['A', leafs, comboGlyph, 'and', rocket]); + }); + + it('should account for regional symbols', () => { + const pair = flag.match(/\ud83c[\udde6-\uddff]/g); + const regionals = pair.join(' '); + + expect(wordsWithNumbers(flag)).toEqual([flag]); + expect(wordsWithNumbers(regionals)).toEqual([pair[0], pair[1]]); + }); + + it('should account for variation selectors', () => { + expect(wordsWithNumbers(heart)).toEqual([heart]); + }); + + it('should match lone surrogates', () => { + const pair = hearts.split(''); + const surrogates = `${pair[0]} ${pair[1]}`; + + expect(wordsWithNumbers(surrogates)).toEqual([]); + }); +}); + +describe('snakeCase snakeCaseWithNumbers', () => { + const caseMethods = { + snakeCase, + snakeCaseWithNumbers, + }; + + _.each(['snakeCase', 'snakeCaseWithNumbers'], (caseName) => { + const methodName = caseName; + const func = caseMethods[methodName]; + + const strings = [ + 'foo bar', + 'Foo bar', + 'foo Bar', + 'Foo Bar', + 'FOO BAR', + 'fooBar', + '--foo-bar--', + '__foo_bar__', + ]; + + const converted = (function () { + switch (caseName) { + case 'snakeCase': + return 'foo_bar'; + case 'snakeCaseWithNumbers': + return 'foo_bar'; + } + })(); + + it(`\`_.${methodName}\` should convert \`string\` to ${caseName} case`, () => { + const actual = _.map(strings, (string) => { + const expected = caseName === 'start' && string === 'FOO BAR' ? string : converted; + return func(string) === expected; + }); + expect(actual).toEqual(_.map(strings, stubTrue)); + }); + + it(`\`_.${methodName}\` should handle double-converting strings`, () => { + const actual = _.map(strings, (string) => { + const expected = caseName === 'start' && string === 'FOO BAR' ? string : converted; + return func(func(string)) === expected; + }); + expect(actual).toEqual(_.map(strings, stubTrue)); + }); + + it(`\`_.${methodName}\` should remove contraction apostrophes`, () => { + const postfixes = ['d', 'll', 'm', 're', 's', 't', 've']; + + _.each(["'", '\u2019'], (apos) => { + const actual = _.map(postfixes, (postfix) => + func(`a b${apos}${postfix} c`), + ); + const expected = _.map(postfixes, (postfix) => { + switch (caseName) { + case 'snakeCase': + return `a_b${postfix}_c`; + case 'snakeCaseWithNumbers': + return `a_b${postfix}_c`; + } + }); + expect(actual).toEqual(expected); + }); + }); + + it(`\`_.${methodName}\` should remove Latin mathematical operators`, () => { + const actual = _.map(['\xd7', '\xf7'], func); + expect(actual).toEqual(['', '']); + }); + + it(`\`_.${methodName}\` should coerce \`string\` to a string`, () => { + const string = 'foo bar'; + expect(func(Object(string))).toBe(converted); + expect(func({toString: _.constant(string)})).toBe(converted); + }); + + it(`\`_.${methodName}\` should return an empty string for empty values`, () => { + const values = [, null, undefined, '']; + const expected = _.map(values, stubString); + + const actual = _.map(values, (value, index) => + index ? func(value) : func(), + ); + + expect(actual).toEqual(expected); + }); + }); +}); \ No newline at end of file diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 83b24aee15..9099566c60 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -1,7 +1,5 @@ const _ = require("lodash"); -const util = require("util"); - const { input, output } = require(`./data/warehouse/events.js`); const { opInput, @@ -21,6 +19,7 @@ const { const { validTimestamp } = require("../../src/warehouse/util.js"); +const {transformTableName, transformColumnName} = require("../../src/warehouse/v1/util"); const {isBlank} = require("../../src/warehouse/config/helpers.js"); const version = "v0"; @@ -1010,28 +1009,6 @@ describe("Add receivedAt for events missing it", () => { }); describe("Integration options", () => { - describe("Destination config options", () => { - destConfig.scenarios().forEach(scenario => { - it(scenario.name, () => { - if (scenario.skipUsersTable !== null) { - scenario.event.destination.Config.skipUsersTable = scenario.skipUsersTable - } - if (scenario.skipTracksTable !== null) { - scenario.event.destination.Config.skipTracksTable = scenario.skipTracksTable - } - - transformers.forEach((transformer, index) => { - const received = transformer.process(scenario.event); - expect(received).toHaveLength(scenario.expected.length); - for (const i in received) { - const evt = received[i]; - expect(evt.data.id ? evt.data.id : evt.data.ID).toEqual(scenario.expected[i].id); - expect(evt.metadata.table.toLowerCase()).toEqual(scenario.expected[i].table); - } - }); - }); - }); - }); describe("track", () => { it("should generate two events for every track call", () => { const i = opInput("track"); @@ -1235,6 +1212,306 @@ describe("isBlank", () => { } }); +describe("Destination config", () => { + describe("skipUsersTable, skipTracksTable", () => { + destConfig.scenarios().forEach(scenario => { + it(scenario.name, () => { + if (scenario.skipUsersTable !== null) { + scenario.event.destination.Config.skipUsersTable = scenario.skipUsersTable + } + if (scenario.skipTracksTable !== null) { + scenario.event.destination.Config.skipTracksTable = scenario.skipTracksTable + } + + transformers.forEach((transformer, index) => { + const received = transformer.process(scenario.event); + expect(received).toHaveLength(scenario.expected.length); + for (const i in received) { + const evt = received[i]; + expect(evt.data.id ? evt.data.id : evt.data.ID).toEqual(scenario.expected[i].id); + expect(evt.metadata.table.toLowerCase()).toEqual(scenario.expected[i].table); + } + }); + }); + }); + }); + + describe('allowUsersContextTraits, underscoreDivideNumbers', () => { + describe("old destinations", () => { + it('with allowUsersContextTraits', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: { + allowUsersContextTraits: true + } + }, + message: { + context: { + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + }, + traits: { + lastname: "Mouse" + }, + type: "identify", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + const events = [output[0], output[1]]; // identifies and users event + const traitsToCheck = { + 'city': 'Disney', + 'country': 'USA', + 'email': 'mickey@disney.com', + 'firstname': 'Mickey' + }; + events.forEach(event => { + Object.entries(traitsToCheck).forEach(([trait, value]) => { + expect(event.data[integrationCasedString(integrations[index], trait)]).toEqual(value); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + }); + }); + }); + }); + + it('with underscoreDivideNumbers', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: { + underscoreDivideNumbers: true + }, + }, + message: { + context: { + 'attribute v3': 'some-value' + }, + event: "button clicked v2", + type: "track", + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v_2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); + }); + }); + }); + describe("new destinations", () => { + it('without allowUsersContextTraits', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: {} + }, + message: { + context: { + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + }, + traits: { + lastname: "Mouse" + }, + type: "identify", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const received = transformer.process(event); + const events = [received[0], received[1]]; // identifies and users event + const traitsToCheck = { + 'city': 'Disney', + 'country': 'USA', + 'email': 'mickey@disney.com', + 'firstname': 'Mickey' + }; + events.forEach(event => { + Object.entries(traitsToCheck).forEach(([trait, value]) => { + expect(event.data).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + }); + }); + }); + }); + + it('without underscoreDivideNumbers', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: {}, + }, + message: { + context: { + 'attribute v3': 'some-value' + }, + event: "button clicked v2", + type: "track", + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); + }); + }); + }); + }); +}); + +describe("validTimestamp", () => { + const testCases = [ + { + name: "undefined input should return false", + input: undefined, + expected: false, + }, + { + name: "negative year and time input should return false #1", + input: '-0001-11-30T00:00:00+0000', + expected: false, + }, + { + name: "negative year and time input should return false #2", + input: '-2023-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "negative year and time input should return false #3", + input: '-1900-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "positive year and time input should return false", + input: '+2023-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "valid timestamp input should return true", + input: '2023-06-14T05:23:59.244Z', + expected: true, + }, + { + name: "non-date string input should return false", + input: 'abc', + expected: false, + }, + { + name: "malicious string input should return false", + input: '%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216Windows%u2216win%u002ein', + expected: false, + }, + { + name: "empty string input should return false", + input: '', + expected: false, + }, + { + name: "valid date input should return true", + input: '2023-06-14', + expected: true, + }, + { + name: "time-only input should return false", + input: '05:23:59.244Z', + expected: false, + }, + { + name: "non-string input should return false", + input: {abc: 123}, + expected: false, + }, + { + name: "object with toString method input should return false", + input: { + toString: '2023-06-14T05:23:59.244Z' + }, + expected: false, + }, + ]; + for (const testCase of testCases) { + it(`should return ${testCase.expected} for ${testCase.name}`, () => { + expect(validTimestamp(testCase.input)).toEqual(testCase.expected); + }); + } +}); + +describe("isBlank", () => { + const testCases = [ + { + name: "null", + input: null, + expected: true + }, + { + name: "empty string", + input: "", + expected: true + }, + { + name: "non-empty string", + input: "test", + expected: false + }, + { + name: "numeric value", + input: 1634762544, + expected: false + }, + { + name: "object with toString property", + input: { + toString: '2023-06-14T05:23:59.244Z' + }, + expected: false + }, + ]; + for (const testCase of testCases) { + it(`should return ${testCase.expected} for ${testCase.name}`, () => { + expect(isBlank(testCase.input)).toEqual(testCase.expected); + }); + } +}); + describe("context traits", () => { const testCases = [ { @@ -1421,3 +1698,651 @@ describe("group traits", () => { }); } }) + +describe("transformColumnName", () => { + describe('with Blendo Casing', () => { + const testCases = [ + { + description: 'should convert special characters other than "\\" or "$" to underscores', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'column@Name$1', + expected: 'column_name$1', + }, + { + description: 'should add underscore if name does not start with an alphabet or underscore', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: '1CComega', + expected: '_1ccomega', + }, + { + description: 'should handle non-ASCII characters by converting to underscores', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'Cízǔ', + expected: 'c_z_', + }, + { + description: 'should transform CamelCase123Key to camelcase123key', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'CamelCase123Key', + expected: 'camelcase123key', + }, + { + description: 'should preserve "\\" and "$" characters', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'path to $1,00,000', + expected: 'path_to_$1_00_000', + }, + { + description: 'should handle a mix of characters, numbers, and special characters', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'CamelCase123Key_with$special\\chars', + expected: 'camelcase123key_with$special\\chars', + }, + { + description: 'should limit length to 63 characters for postgres provider', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'a'.repeat(70), + expected: 'a'.repeat(63), + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformColumnName(options, input); + expect(result).toBe(expected); + }); + }); + }); + + describe('without Blendo Casing (underscoreDivideNumbers=true)', () => { + const testCases = [ + { + description: 'should remove symbols and join continuous letters and numbers with a single underscore', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '&4yasdfa(84224_fs9##_____*3q', + expected: '_4_yasdfa_84224_fs_9_3_q', + }, + { + description: 'should transform "omega" to "omega"', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega', + expected: 'omega', + }, + { + description: 'should transform "omega v2" to "omega_v_2"', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega v2', + expected: 'omega_v_2', + }, + { + description: 'should prepend underscore if name starts with a number', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '9mega', + expected: '_9_mega', + }, + { + description: 'should remove trailing special characters', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'mega&', + expected: 'mega', + }, + { + description: 'should replace special character in the middle with underscore', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'ome$ga', + expected: 'ome_ga', + }, + { + description: 'should not remove trailing $ character', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega$', + expected: 'omega', + }, + { + description: 'should handle spaces and special characters by converting to underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'ome_ ga', + expected: 'ome_ga', + }, + { + description: 'should handle multiple underscores and hyphens by reducing to single underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '9mega________-________90', + expected: '_9_mega_90', + }, + { + description: 'should handle non-ASCII characters by converting them to underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'Cízǔ', + expected: 'c_z', + }, + { + description: 'should transform CamelCase123Key to camel_case_123_key', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'CamelCase123Key', + expected: 'camel_case_123_key', + }, + { + description: 'should handle numbers and commas in the input', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'path to $1,00,000', + expected: 'path_to_1_00_000', + }, + { + description: 'should return an empty string if input contains no valid characters', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '@#$%', + expected: '', + }, + { + description: 'should keep underscores between letters and numbers', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'test123', + expected: 'test_123', + }, + { + description: 'should keep multiple underscore-number sequences', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'abc123def456', + expected: 'abc_123_def_456', + }, + { + description: 'should keep multiple underscore-number sequences', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'abc_123_def_456', + expected: 'abc_123_def_456', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformColumnName(options, input); + expect(result).toBe(expected); + }); + }); + }); + + describe('without Blendo Casing (underscoreDivideNumbers=false)', () => { + const testCases = [ + { + description: 'should remove symbols and join continuous letters and numbers with a single underscore', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: '&4yasdfa(84224_fs9##_____*3q', + expected: '_4yasdfa_84224_fs9_3q', + }, + { + description: 'should transform "omega" to "omega"', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'omega', + expected: 'omega', + }, + { + description: 'should transform "omega v2" to "omega_v_2"', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'omega v2', + expected: 'omega_v2', + }, + { + description: 'should prepend underscore if name starts with a number', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: '9mega', + expected: '_9mega', + }, + { + description: 'should remove trailing special characters', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'mega&', + expected: 'mega', + }, + { + description: 'should replace special character in the middle with underscore', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'ome$ga', + expected: 'ome_ga', + }, + { + description: 'should not remove trailing $ character', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega$', + expected: 'omega', + }, + { + description: 'should handle spaces and special characters by converting to underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'ome_ ga', + expected: 'ome_ga', + }, + { + description: 'should handle multiple underscores and hyphens by reducing to single underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: '9mega________-________90', + expected: '_9mega_90', + }, + { + description: 'should handle non-ASCII characters by converting them to underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'Cízǔ', + expected: 'c_z', + }, + { + description: 'should transform CamelCase123Key to camel_case_123_key', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'CamelCase123Key', + expected: 'camel_case123_key', + }, + { + description: 'should handle numbers and commas in the input', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'path to $1,00,000', + expected: 'path_to_1_00_000', + }, + { + description: 'should return an empty string if input contains no valid characters', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: '@#$%', + expected: '', + }, + { + description: 'should keep underscores between letters and numbers', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'test123', + expected: 'test123', + }, + { + description: 'should keep multiple underscore-number sequences', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'abc123def456', + expected: 'abc123_def456', + }, + { + description: 'should keep multiple underscore-number sequences', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'abc_123_def_456', + expected: 'abc_123_def_456', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformColumnName(options, input); + expect(result).toBe(expected); + }); + }); + }); +}) + +describe("transformTableName", () => { + describe('with Blendo Casing', () => { + const testCases = [ + { + description: 'should convert name to Blendo casing (lowercase) when Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'TableName123', + expected: 'tablename123', + }, + { + description: 'should trim spaces and convert to Blendo casing (lowercase) when Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: ' TableName ', + expected: 'tablename', + }, + { + description: 'should return an empty string when input is empty and Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: '', + expected: '', + }, + { + description: 'should handle names with special characters and convert to Blendo casing (lowercase)', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'Table@Name!', + expected: 'table@name!', + }, + { + description: 'should convert a mixed-case name to Blendo casing (lowercase) when Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'CaMeLcAsE', + expected: 'camelcase', + }, + { + description: 'should keep an already lowercase name unchanged with Blendo casing enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'lowercase', + expected: 'lowercase', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformTableName(options, input); + expect(result).toBe(expected); + }); + }); + }); + + describe('without Blendo Casing (underscoreDivideNumbers=true)', () => { + const testCases = [ + { + description: 'should remove symbols and join continuous letters and numbers with a single underscore', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '&4yasdfa(84224_fs9##_____*3q', + expected: '_4_yasdfa_84224_fs_9_3_q', + }, + { + description: 'should transform "omega" to "omega"', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega', + expected: 'omega', + }, + { + description: 'should transform "omega v2" to "omega_v_2"', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega v2', + expected: 'omega_v_2', + }, + { + description: 'should prepend underscore if name starts with a number', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '9mega', + expected: '_9_mega', + }, + { + description: 'should remove trailing special characters', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'mega&', + expected: 'mega', + }, + { + description: 'should replace special character in the middle with underscore', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'ome$ga', + expected: 'ome_ga', + }, + { + description: 'should not remove trailing $ character', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega$', + expected: 'omega', + }, + { + description: 'should handle spaces and special characters by converting to underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'ome_ ga', + expected: 'ome_ga', + }, + { + description: 'should handle multiple underscores and hyphens by reducing to single underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '9mega________-________90', + expected: '_9_mega_90', + }, + { + description: 'should handle non-ASCII characters by converting them to underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'Cízǔ', + expected: 'c_z', + }, + { + description: 'should transform CamelCase123Key to camel_case_123_key', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'CamelCase123Key', + expected: 'camel_case_123_key', + }, + { + description: 'should handle numbers and commas in the input', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'path to $1,00,000', + expected: 'path_to_1_00_000', + }, + { + description: 'should return an empty string if input contains no valid characters', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '@#$%', + expected: '', + }, + { + description: 'should keep underscores between letters and numbers', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'test123', + expected: 'test_123', + }, + { + description: 'should keep multiple underscore-number sequences', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'abc123def456', + expected: 'abc_123_def_456', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformTableName(options, input); + expect(result).toBe(expected); + }); + }); + }); + + describe('without Blendo Casing (underscoreDivideNumbers=false)', () => { + const testCases = [ + { + description: 'should remove symbols and join continuous letters and numbers with a single underscore', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: '&4yasdfa(84224_fs9##_____*3q', + expected: '_4yasdfa_84224_fs9_3q', + }, + { + description: 'should transform "omega" to "omega"', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'omega', + expected: 'omega', + }, + { + description: 'should transform "omega v2" to "omega_v_2"', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'omega v2', + expected: 'omega_v2', + }, + { + description: 'should prepend underscore if name starts with a number', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: '9mega', + expected: '_9mega', + }, + { + description: 'should remove trailing special characters', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'mega&', + expected: 'mega', + }, + { + description: 'should replace special character in the middle with underscore', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'ome$ga', + expected: 'ome_ga', + }, + { + description: 'should not remove trailing $ character', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega$', + expected: 'omega', + }, + { + description: 'should handle spaces and special characters by converting to underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'ome_ ga', + expected: 'ome_ga', + }, + { + description: 'should handle multiple underscores and hyphens by reducing to single underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: '9mega________-________90', + expected: '_9mega_90', + }, + { + description: 'should handle non-ASCII characters by converting them to underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'Cízǔ', + expected: 'c_z', + }, + { + description: 'should transform CamelCase123Key to camel_case_123_key', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'CamelCase123Key', + expected: 'camel_case123_key', + }, + { + description: 'should handle numbers and commas in the input', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'path to $1,00,000', + expected: 'path_to_1_00_000', + }, + { + description: 'should return an empty string if input contains no valid characters', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: '@#$%', + expected: '', + }, + { + description: 'should keep underscores between letters and numbers', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'test123', + expected: 'test123', + }, + { + description: 'should keep multiple underscore-number sequences', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'abc123def456', + expected: 'abc123_def456', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformTableName(options, input); + expect(result).toBe(expected); + }); + }); + }); +}); \ No newline at end of file diff --git a/test/integrations/destinations/gcs_datalake/processor/data.ts b/test/integrations/destinations/gcs_datalake/processor/data.ts index 46c7788709..26f438758d 100644 --- a/test/integrations/destinations/gcs_datalake/processor/data.ts +++ b/test/integrations/destinations/gcs_datalake/processor/data.ts @@ -62,6 +62,8 @@ export const data = [ syncFrequency: '30', tableSuffix: '', timeWindowLayout: '2006/01/02/15', + allowUsersContextTraits: true, + underscoreDivideNumbers: true, }, Enabled: true, },