chore: address review comments

rudderlabs · Sep 10, 2024 · 77673c2 · 77673c2
1 parent 7dcbd58
commit 77673c2
Show file tree

Hide file tree

Showing 24 changed files with 1,297 additions and 565 deletions.
diff --git a/src/warehouse/index.js b/src/warehouse/index.js
@@ -563,7 +563,6 @@ function processWarehouseMessage(message, options) {
     message.integrations && message.integrations[options.provider.toUpperCase()]
       ? message.integrations[options.provider.toUpperCase()].options
       : {};
-
   const responses = [];
   const eventType = message.type?.toLowerCase();
   const skipTracksTable =
@@ -585,13 +584,6 @@ function processWarehouseMessage(message, options) {
   // For older destinations, it will come as true, and for new destinations this config will not be present, which means we will treat it as false.
   const allowUsersContextTraits = options.destConfig?.allowUsersContextTraits || false;
 
-  // allowEventContextTraits when set to true, if context.traits.* is present, it will be added as context_traits_* in the events table,
-  // e.g., for context.traits.name, context_traits_name will be added to the events table.
-  // allowEventContextTraits when set to false, if context.traits.* is present, nothing will be added in the events table.
-  // e.g., for context.traits.name, nothing will be added to the events table.
-  // For older destinations, it will come as true, and for new destinations this config will not be present, which means we will treat it as false.
-  const allowEventContextTraits = options.destConfig?.allowEventContextTraits || false;
-
   addJsonKeysToOptions(options);
 
   if (isBlank(message.messageId)) {
@@ -805,14 +797,6 @@ function processWarehouseMessage(message, options) {
         ...trackProps,
         ...commonProps,
       };
-      if (!allowEventContextTraits) {
-        // remove all context traits from eventTableEvent
-        Object.keys(eventTableEvent).forEach((key) => {
-          if (key.toLowerCase().startsWith('context_traits_')) {
-            delete eventTableEvent[key];
-          }
-        });
-      }
       const eventTableMetadata = {
         table: excludeRudderCreatedTableNames(
           utils.safeTableName(

diff --git a/src/warehouse/snakecase/snakecase.js b/src/warehouse/snakecase/snakecase.js
@@ -0,0 +1,37 @@
+import { toString } from 'lodash';
+import { unicodeWords, unicodeWordsWithNumbers } from './unicodeWords';
+
+const hasUnicodeWord = RegExp.prototype.test.bind(
+  /[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/,
+);
+
+/** Used to match words composed of alphanumeric characters. */
+// eslint-disable-next-line no-control-regex
+const reAsciiWord = /[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g;
+
+function asciiWords(string) {
+  return string.match(reAsciiWord);
+}
+
+function words(string) {
+  const result = hasUnicodeWord(string) ? unicodeWords(string) : asciiWords(string);
+  return result || [];
+}
+
+function wordsWithNumbers(string) {
+  const result = hasUnicodeWord(string) ? unicodeWordsWithNumbers(string) : asciiWords(string);
+  return result || [];
+}
+
+const snakeCase = (string) =>
+  words(toString(string).replace(/['\u2019]/g, '')).reduce(
+    (result, word, index) => result + (index ? '_' : '') + word.toLowerCase(),
+    '',
+  );
+const snakeCaseWithNumbers = (string) =>
+  wordsWithNumbers(toString(string).replace(/['\u2019]/g, '')).reduce(
+    (result, word, index) => result + (index ? '_' : '') + word.toLowerCase(),
+    '',
+  );
+
+export { words, wordsWithNumbers, snakeCase, snakeCaseWithNumbers };
diff --git a/src/warehouse/snakecase/unicodeWords.js b/src/warehouse/snakecase/unicodeWords.js
@@ -0,0 +1,94 @@
+/** Used to compose unicode character classes. */
+const rsAstralRange = '\\ud800-\\udfff';
+const rsComboMarksRange = '\\u0300-\\u036f';
+const reComboHalfMarksRange = '\\ufe20-\\ufe2f';
+const rsComboSymbolsRange = '\\u20d0-\\u20ff';
+const rsComboMarksExtendedRange = '\\u1ab0-\\u1aff';
+const rsComboMarksSupplementRange = '\\u1dc0-\\u1dff';
+const rsComboRange =
+  rsComboMarksRange +
+  reComboHalfMarksRange +
+  rsComboSymbolsRange +
+  rsComboMarksExtendedRange +
+  rsComboMarksSupplementRange;
+const rsDingbatRange = '\\u2700-\\u27bf';
+const rsLowerRange = 'a-z\\xdf-\\xf6\\xf8-\\xff';
+const rsMathOpRange = '\\xac\\xb1\\xd7\\xf7';
+const rsNonCharRange = '\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf';
+const rsPunctuationRange = '\\u2000-\\u206f';
+const rsSpaceRange =
+  ' \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000';
+const rsUpperRange = 'A-Z\\xc0-\\xd6\\xd8-\\xde';
+const rsVarRange = '\\ufe0e\\ufe0f';
+const rsBreakRange = rsMathOpRange + rsNonCharRange + rsPunctuationRange + rsSpaceRange;
+
+/** Used to compose unicode capture groups. */
+const rsApos = "['\u2019]";
+const rsBreak = `[${rsBreakRange}]`;
+const rsCombo = `[${rsComboRange}]`;
+const rsDigit = '\\d';
+const rsDingbat = `[${rsDingbatRange}]`;
+const rsLower = `[${rsLowerRange}]`;
+const rsMisc = `[^${rsAstralRange}${rsBreakRange + rsDigit + rsDingbatRange + rsLowerRange + rsUpperRange}]`;
+const rsFitz = '\\ud83c[\\udffb-\\udfff]';
+const rsModifier = `(?:${rsCombo}|${rsFitz})`;
+const rsNonAstral = `[^${rsAstralRange}]`;
+const rsRegional = '(?:\\ud83c[\\udde6-\\uddff]){2}';
+const rsSurrPair = '[\\ud800-\\udbff][\\udc00-\\udfff]';
+const rsUpper = `[${rsUpperRange}]`;
+const rsZWJ = '\\u200d';
+
+/** Used to compose unicode regexes. */
+const rsMiscLower = `(?:${rsLower}|${rsMisc})`;
+const rsMiscUpper = `(?:${rsUpper}|${rsMisc})`;
+const rsOptContrLower = `(?:${rsApos}(?:d|ll|m|re|s|t|ve))?`;
+const rsOptContrUpper = `(?:${rsApos}(?:D|LL|M|RE|S|T|VE))?`;
+const reOptMod = `${rsModifier}?`;
+const rsOptVar = `[${rsVarRange}]?`;
+const rsOptJoin = `(?:${rsZWJ}(?:${[rsNonAstral, rsRegional, rsSurrPair].join('|')})${rsOptVar + reOptMod})*`;
+const rsOrdLower = '\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])';
+const rsOrdUpper = '\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])';
+const rsSeq = rsOptVar + reOptMod + rsOptJoin;
+const rsEmoji = `(?:${[rsDingbat, rsRegional, rsSurrPair].join('|')})${rsSeq}`;
+
+const reUnicodeWords = RegExp(
+  [
+    `${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`, // Regular words, lowercase letters followed by optional contractions
+    `${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`, // Miscellaneous uppercase characters with optional contractions
+    `${rsUpper}?${rsMiscLower}+${rsOptContrLower}`, // Miscellaneous lowercase sequences with optional contractions
+    `${rsUpper}+${rsOptContrUpper}`, // All uppercase words with optional contractions (e.g., "THIS")
+    rsOrdUpper, // Ordinals for uppercase (e.g., "1ST", "2ND")
+    rsOrdLower, // Ordinals for lowercase (e.g., "1st", "2nd")
+    `${rsDigit}+`, // Pure digits (e.g., "123")
+    rsEmoji, // Emojis (e.g., 😀, ❤️)
+  ].join('|'),
+  'g',
+);
+
+const reUnicodeWordsWithNumbers = RegExp(
+  [
+    `${rsUpper}?${rsLower}+${rsDigit}+`, // Lowercase letters followed by digits (e.g., "abc123")
+    `${rsUpper}+${rsDigit}+`, // Uppercase letters followed by digits (e.g., "ABC123")
+    `${rsDigit}+${rsUpper}?${rsLower}+`, // Digits followed by lowercase letters (e.g., "123abc")
+    `${rsDigit}+${rsUpper}+`, // Digits followed by uppercase letters (e.g., "123ABC")
+    `${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`, // Regular words, lowercase letters followed by optional contractions
+    `${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`, // Miscellaneous uppercase characters with optional contractions
+    `${rsUpper}?${rsMiscLower}+${rsOptContrLower}`, // Miscellaneous lowercase sequences with optional contractions
+    `${rsUpper}+${rsOptContrUpper}`, // All uppercase words with optional contractions (e.g., "THIS")
+    rsOrdUpper, // Ordinals for uppercase (e.g., "1ST", "2ND")
+    rsOrdLower, // Ordinals for lowercase (e.g., "1st", "2nd")
+    `${rsDigit}+`, // Pure digits (e.g., "123")
+    rsEmoji, // Emojis (e.g., 😀, ❤️)
+  ].join('|'),
+  'g',
+);
+
+function unicodeWords(string) {
+  return string.match(reUnicodeWords);
+}
+
+function unicodeWordsWithNumbers(string) {
+  return string.match(reUnicodeWordsWithNumbers);
+}
+
+export { unicodeWords, unicodeWordsWithNumbers };
diff --git a/src/warehouse/util.js b/src/warehouse/util.js
@@ -1,4 +1,3 @@
-const _ = require('lodash');
 const get = require('get-value');
 
 const v1 = require('./v1/util');

diff --git a/src/warehouse/v1/util.js b/src/warehouse/v1/util.js
@@ -1,8 +1,7 @@
-const _ = require('lodash');
-
 const reservedANSIKeywordsMap = require('../config/ReservedKeywords.json');
 const { isDataLakeProvider } = require('../config/helpers');
 const { TransformationError } = require('@rudderstack/integrations-lib');
+const { snakeCase, snakeCaseWithNumbers } = require('../snakecase/snakecase');
 
 function safeTableName(options, name = '') {
   const { provider } = options;
@@ -104,14 +103,17 @@ function transformName(options, provider, name = '') {
   if (extractedValue !== '') {
     extractedValues.push(extractedValue);
   }
+  const underscoreDivideNumbers = options?.underscoreDivideNumbers || false;
+  const snakeCaseFn = underscoreDivideNumbers ? snakeCase : snakeCaseWithNumbers;
+
   let key = extractedValues.join('_');
   if (name.startsWith('_')) {
     // do not remove leading underscores to allow esacaping rudder keywords with underscore
     // _timestamp -> _timestamp
     // __timestamp -> __timestamp
-    key = name.match(/^_*/)[0] + _.snakeCase(key.replace(/^_*/, ''));
+    key = name.match(/^_*/)[0] + snakeCaseFn(key.replace(/^_*/, ''));
   } else {
-    key = _.snakeCase(key);
+    key = snakeCaseFn(key);
   }
 
   if (key !== '' && key.charCodeAt(0) >= 48 && key.charCodeAt(0) <= 57) {
@@ -120,9 +122,6 @@ function transformName(options, provider, name = '') {
   if (provider === 'postgres') {
     key = key.substr(0, 63);
   }
-  if (!options?.underscoreDivideNumbers) {
-    key = key.replace(/(\w)_(\d+)/g, '$1$2');
-  }
   return key;
 }
 

diff --git a/test/__tests__/data/warehouse/events.js b/test/__tests__/data/warehouse/events.js
@@ -10,7 +10,6 @@ const sampleEvents = {
           prefixProperties: true,
           useNativeSDK: false,
           allowUsersContextTraits: true,
-          allowEventContextTraits: true,
           underscoreDivideNumbers: true
         },
         DestinationDefinition: {
@@ -519,7 +518,6 @@ const sampleEvents = {
           prefixProperties: true,
           useNativeSDK: false,
           allowUsersContextTraits: true,
-          allowEventContextTraits: true,
           underscoreDivideNumbers: true
         },
         DestinationDefinition: {
@@ -1034,7 +1032,6 @@ const sampleEvents = {
           prefixProperties: true,
           useNativeSDK: false,
           allowUsersContextTraits: true,
-          allowEventContextTraits: true,
           underscoreDivideNumbers: true
         },
         DestinationDefinition: {
@@ -1294,7 +1291,6 @@ const sampleEvents = {
           trackCategorisedPages: true,
           trackNamedPages: false,
           allowUsersContextTraits: true,
-          allowEventContextTraits: true,
           underscoreDivideNumbers: true
         },
         Enabled: true
@@ -1546,7 +1542,6 @@ const sampleEvents = {
           trackCategorisedPages: false,
           trackNamedPages: false,
           allowUsersContextTraits: true,
-          allowEventContextTraits: true,
           underscoreDivideNumbers: true
         },
         Enabled: true
@@ -1769,7 +1764,6 @@ const sampleEvents = {
           prefixProperties: true,
           useNativeSDK: false,
           allowUsersContextTraits: true,
-          allowEventContextTraits: true,
           underscoreDivideNumbers: true
         },
         DestinationDefinition: {
@@ -2036,7 +2030,6 @@ const sampleEvents = {
           prefixProperties: true,
           useNativeSDK: false,
           allowUsersContextTraits: true,
-          allowEventContextTraits: true,
           underscoreDivideNumbers: true
         },
         DestinationDefinition: {

diff --git a/test/__tests__/data/warehouse/integration_options_events.js b/test/__tests__/data/warehouse/integration_options_events.js
@@ -22,7 +22,6 @@ const sampleEvents = {
         Config: {
           jsonPaths: " testMap.nestedMap, testArray",
           allowUsersContextTraits: true,
-          allowEventContextTraits: true,
           underscoreDivideNumbers: true
         }
       },
@@ -736,7 +735,6 @@ const sampleEvents = {
       destination: {
         Config: {
           allowUsersContextTraits: true,
-          allowEventContextTraits: true,
           underscoreDivideNumbers: true
         }
       },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },

diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js
@@ -3,7 +3,6 @@ module.exports = {
         destination: {
             Config: {
                 allowUsersContextTraits: true,
-                allowEventContextTraits: true,
                 underscoreDivideNumbers: true
             }
         },