diff --git a/src/warehouse/index.js b/src/warehouse/index.js index f62f02af79..3ce2450685 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -563,6 +563,7 @@ function processWarehouseMessage(message, options) { message.integrations && message.integrations[options.provider.toUpperCase()] ? message.integrations[options.provider.toUpperCase()].options : {}; + const responses = []; const eventType = message.type?.toLowerCase(); const skipTracksTable = @@ -572,6 +573,25 @@ function processWarehouseMessage(message, options) { const skipReservedKeywordsEscaping = options.integrationOptions.skipReservedKeywordsEscaping || false; + // underscoreDivideNumbers when set to false, if a column has a format like "_v_3_", it will be formatted to "_v3_" + // underscoreDivideNumbers when set to true, if a column has a format like "_v_3_", we keep it like that + // For older destinations, it will come as true and for new destinations this config will not be present which means we will treat it as false. + options.underscoreDivideNumbers = options.destConfig?.underscoreDivideNumbers || false; + + // allowUsersContextTraits when set to true, if context.traits.* is present, it will be added as context_traits_* and *, + // e.g., for context.traits.name, context_traits_name and name will be added to the users table. + // allowUsersContextTraits when set to false, if context.traits.* is present, it will be added only as context_traits_* + // e.g., for context.traits.name, only context_traits_name will be added to the users table. + // For older destinations, it will come as true and for new destinations this config will not be present which means we will treat it as false. + const allowUsersContextTraits = options.destConfig?.allowUsersContextTraits || false; + + // allowEventContextTraits when set to true, if context.traits.* is present, it will be added as context_traits_* in the events table + // e.g., for context.traits.name, context_traits_name will be added to the events table. + // allowEventContextTraits when set to false, if context.traits.* is present, nothing will be added in events table. + // e.g., for context.traits.name, nothing will be added to the events table. + // For older destinations, it will come as true and for new destinations this config will not be present which means we will treat it as false. + const allowEventContextTraits = options.destConfig?.allowEventContextTraits || false; + addJsonKeysToOptions(options); if (isBlank(message.messageId)) { @@ -785,6 +805,14 @@ function processWarehouseMessage(message, options) { ...trackProps, ...commonProps, }; + if (!allowEventContextTraits) { + // remove all context traits from eventTableEvent + Object.keys(eventTableEvent).forEach((key) => { + if (key.toLowerCase().startsWith('context_traits_')) { + delete eventTableEvent[key]; + } + }); + } const eventTableMetadata = { table: excludeRudderCreatedTableNames( utils.safeTableName( @@ -827,16 +855,18 @@ function processWarehouseMessage(message, options) { `${eventType + '_userProperties_'}`, 2, ); - setDataFromInputAndComputeColumnTypes( - utils, - eventType, - commonProps, - message.context ? message.context.traits : {}, - commonColumnTypes, - options, - `${eventType + '_context_traits_'}`, - 3, - ); + if (allowUsersContextTraits) { + setDataFromInputAndComputeColumnTypes( + utils, + eventType, + commonProps, + message.context ? message.context.traits : {}, + commonColumnTypes, + options, + `${eventType + '_context_traits_'}`, + 3, + ); + } setDataFromInputAndComputeColumnTypes( utils, eventType, diff --git a/src/warehouse/util.js b/src/warehouse/util.js index b4b22721fd..a2c6271d64 100644 --- a/src/warehouse/util.js +++ b/src/warehouse/util.js @@ -1,7 +1,6 @@ const _ = require('lodash'); const get = require('get-value'); -const v0 = require('./v0/util'); const v1 = require('./v1/util'); const { PlatformError, InstrumentationError } = require('@rudderstack/integrations-lib'); const { isBlank } = require('./config/helpers'); @@ -112,14 +111,7 @@ function validTimestamp(input) { } function getVersionedUtils(schemaVersion) { - switch (schemaVersion) { - case 'v0': - return v0; - case 'v1': - return v1; - default: - return v1; - } + return v1; } function isRudderSourcesEvent(event) { diff --git a/src/warehouse/v0/util.js b/src/warehouse/v0/util.js deleted file mode 100644 index 5917f8ea08..0000000000 --- a/src/warehouse/v0/util.js +++ /dev/null @@ -1,87 +0,0 @@ -const reservedANSIKeywordsMap = require('../config/ReservedKeywords.json'); -const { isDataLakeProvider } = require('../config/helpers'); - -const toSnakeCase = (str) => { - if (!str) { - return ''; - } - return String(str) - .replace(/^[^A-Za-z0-9]*|[^A-Za-z0-9]*$/g, '') - .replace(/([a-z])([A-Z])/g, (m, a, b) => `${a}_${b.toLowerCase()}`) - .replace(/[^A-Za-z0-9]+|_+/g, '_') - .toLowerCase(); -}; - -function toSafeDBString(provider, name = '') { - let parsedStr = name; - if (parseInt(name[0], 10) >= 0) { - parsedStr = `_${name}`; - } - parsedStr = parsedStr.replace(/[^a-zA-Z0-9_]+/g, ''); - if (isDataLakeProvider(provider)) { - return parsedStr; - } - switch (provider) { - case 'postgres': - return parsedStr.substr(0, 63); - default: - return parsedStr.substr(0, 127); - } -} - -function safeTableName(provider, name = '') { - let tableName = name; - if (tableName === '') { - tableName = 'STRINGEMPTY'; - } - if (provider === 'snowflake') { - tableName = tableName.toUpperCase(); - } - if (provider === 'rs' || isDataLakeProvider(provider)) { - tableName = tableName.toLowerCase(); - } - if (provider === 'postgres') { - tableName = tableName.substr(0, 63); - tableName = tableName.toLowerCase(); - } - if (reservedANSIKeywordsMap[provider.toUpperCase()][tableName.toUpperCase()]) { - tableName = `_${tableName}`; - } - return tableName; -} - -function safeColumnName(provider, name = '') { - let columnName = name; - if (columnName === '') { - columnName = 'STRINGEMPTY'; - } - if (provider === 'snowflake') { - columnName = columnName.toUpperCase(); - } - if (provider === 'rs' || isDataLakeProvider(provider)) { - columnName = columnName.toLowerCase(); - } - if (provider === 'postgres') { - columnName = columnName.substr(0, 63); - columnName = columnName.toLowerCase(); - } - if (reservedANSIKeywordsMap[provider.toUpperCase()][columnName.toUpperCase()]) { - columnName = `_${columnName}`; - } - return columnName; -} - -function transformTableName(name = '') { - return toSnakeCase(name); -} - -function transformColumnName(provider, name = '') { - return toSafeDBString(provider, name); -} - -module.exports = { - safeColumnName, - safeTableName, - transformColumnName, - transformTableName, -}; diff --git a/src/warehouse/v1/util.js b/src/warehouse/v1/util.js index 1c44a2385e..43cb1d856d 100644 --- a/src/warehouse/v1/util.js +++ b/src/warehouse/v1/util.js @@ -82,7 +82,7 @@ function safeColumnName(options, name = '') { path to $1,00,000 to path_to_1_00_000 return an empty string if it couldn't find a char if its ascii value doesnt belong to numbers or english alphabets */ -function transformName(provider, name = '') { +function transformName(options, provider, name = '') { const extractedValues = []; let extractedValue = ''; for (let i = 0; i < name.length; i += 1) { @@ -120,6 +120,9 @@ function transformName(provider, name = '') { if (provider === 'postgres') { key = key.substr(0, 63); } + if (!options?.underscoreDivideNumbers) { + key = key.replace(/(\w)_(\d+)/g, '$1$2'); + } return key; } @@ -150,7 +153,7 @@ function toBlendoCase(name = '') { function transformTableName(options, name = '') { const useBlendoCasing = options.integrationOptions?.useBlendoCasing || false; - return useBlendoCasing ? toBlendoCase(name) : transformName('', name); + return useBlendoCasing ? toBlendoCase(name) : transformName(options, '', name); } function transformColumnName(options, name = '') { @@ -158,7 +161,7 @@ function transformColumnName(options, name = '') { const useBlendoCasing = options.integrationOptions?.useBlendoCasing || false; return useBlendoCasing ? transformNameToBlendoCase(provider, name) - : transformName(provider, name); + : transformName(options, provider, name); } module.exports = { diff --git a/test/__tests__/data/warehouse/events.js b/test/__tests__/data/warehouse/events.js index ef9cc21096..6338dfc995 100644 --- a/test/__tests__/data/warehouse/events.js +++ b/test/__tests__/data/warehouse/events.js @@ -1,2188 +1,2208 @@ const _ = require("lodash"); -// TODO: also include events with whSchemaVersions v0 const sampleEvents = { - track: { - input: { - destination: { - Config: { - restApiKey: "dummyApiKey", - prefixProperties: true, - useNativeSDK: false - }, - DestinationDefinition: { - DisplayName: "Braze", - ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", - Name: "BRAZE" - }, - Enabled: true, - ID: "1WhcOCGgj9asZu850HvugU2C3Aq", - Name: "Braze", - Transformations: [] - }, - message: { - anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.5" - }, - ip: "0.0.0.0", - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.5" - }, - locale: "en-GB", - os: { - name: "", - version: "" - }, - screen: { - density: 2 - }, - traits: { - city: "Disney", - country: "USA", - email: "mickey@disney.com", - firstname: "Mickey" - }, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" - }, - event: "button clicked", - integrations: { - All: true - }, - messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - originalTimestamp: "2020-01-24T06:29:02.364Z", - properties: { - currency: "USD", - revenue: 50, - stack: { - history: { - errorDetails: [ - { - "message": "Cannot set headers after they are sent to the client", - "toString": "[function]" + track: { + input: { + destination: { + Config: { + restApiKey: "dummyApiKey", + prefixProperties: true, + useNativeSDK: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + DestinationDefinition: { + DisplayName: "Braze", + ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", + Name: "BRAZE" + }, + Enabled: true, + ID: "1WhcOCGgj9asZu850HvugU2C3Aq", + Name: "Braze", + Transformations: [] + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + os: { + name: "", + version: "" + }, + screen: { + density: 2 + }, + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + event: "button clicked", + integrations: { + All: true + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + properties: { + currency: "USD", + revenue: 50, + stack: { + history: { + errorDetails: [ + { + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" + } + ] + } + } + }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "track", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" } - ] } - } }, - receivedAt: "2020-01-24T11:59:02.403+05:30", - request_ip: "[::1]:53708", - sentAt: "2020-01-24T06:29:02.364Z", - timestamp: "2020-01-24T11:59:02.403+05:30", - type: "track", - userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" - }, - request: { - query: { - whSchemaVersion: "v1" + output: { + default: [ + { + metadata: { + table: "tracks", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + event_text: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + event: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + event_text: "button clicked", + id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.364Z", + channel: "web", + event: "button_clicked" + } + }, + { + metadata: { + table: "button_clicked", + columns: { + uuid_ts: "datetime", + currency: "string", + revenue: "int", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + event_text: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + event: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + currency: "USD", + revenue: 50, + stack_history_error_details: [ + { + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" + } + ], + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + event_text: "button clicked", + id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.364Z", + channel: "web", + event: "button_clicked" + } + } + ], + snowflake: [ + { + metadata: { + table: "TRACKS", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "button clicked", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + EVENT: "button_clicked" + } + }, + { + metadata: { + table: "BUTTON_CLICKED", + columns: { + UUID_TS: "datetime", + CURRENCY: "string", + REVENUE: "int", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CURRENCY: "USD", + REVENUE: 50, + STACK_HISTORY_ERROR_DETAILS: [ + { + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" + } + ], + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "button clicked", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + EVENT: "button_clicked", + } + } + ], + s3_datalake: [ + { + metadata: { + table: "tracks", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + event_text: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + _timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + event: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + event_text: "button clicked", + id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.364Z", + _timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.364Z", + channel: "web", + event: "button_clicked" + } + }, + { + metadata: { + table: "button_clicked", + columns: { + uuid_ts: "datetime", + currency: "string", + revenue: "int", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + event_text: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + _timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + event: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + currency: "USD", + revenue: 50, + stack_history_error_details: [ + { + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" + } + ], + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + event_text: "button clicked", + id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.364Z", + _timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.364Z", + channel: "web", + event: "button_clicked" + } + } + ] } - } }, - output: { - default: [ - { - metadata: { - table: "tracks", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - event_text: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - event: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - event_text: "button clicked", - id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.364Z", - timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.364Z", - channel: "web", - event: "button_clicked" - } - }, - { - metadata: { - table: "button_clicked", - columns: { - uuid_ts: "datetime", - currency: "string", - revenue: "int", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - event_text: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - event: "string" + identify: { + input: { + destination: { + Config: { + restApiKey: "dummyApiKey", + prefixProperties: true, + useNativeSDK: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + DestinationDefinition: { + DisplayName: "Braze", + ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", + Name: "BRAZE" + }, + Enabled: true, + ID: "1WhcOCGgj9asZu850HvugU2C3Aq", + Name: "Braze", + Transformations: [] }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - currency: "USD", - revenue: 50, - stack_history_error_details: [ - { - "message": "Cannot set headers after they are sent to the client", - "toString": "[function]" - } - ], - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - event_text: "button clicked", - id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.364Z", - timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.364Z", - channel: "web", - event: "button_clicked" - } - } - ], - snowflake: [ - { - metadata: { - table: "TRACKS", - columns: { - UUID_TS: "datetime", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_TRAITS_CITY: "string", - CONTEXT_TRAITS_COUNTRY: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_TRAITS_FIRSTNAME: "string", - CONTEXT_USER_AGENT: "string", - EVENT_TEXT: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - CHANNEL: "string", - EVENT: "string" + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + os: { + name: "", + version: "" + }, + screen: { + density: 2 + }, + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + traits: { + country: "UK", + lastname: "Mouse" + }, + integrations: { + All: true + }, + messageId: "2536eda4-d638-4c93-8014-8ffe3f083214", + originalTimestamp: "2020-01-24T06:29:02.362Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.363Z", + timestamp: "2020-01-24T11:59:02.402+05:30", + type: "identify", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_TRAITS_CITY: "Disney", - CONTEXT_TRAITS_COUNTRY: "USA", - CONTEXT_TRAITS_EMAIL: "mickey@disney.com", - CONTEXT_TRAITS_FIRSTNAME: "Mickey", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - EVENT_TEXT: "button clicked", - ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - SENT_AT: "2020-01-24T06:29:02.364Z", - TIMESTAMP: "2020-01-24T06:29:02.403Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", - CHANNEL: "web", - EVENT: "button_clicked" - } + request: { + query: { + whSchemaVersion: "v1" + } + } }, - { - metadata: { - table: "BUTTON_CLICKED", - columns: { - UUID_TS: "datetime", - CURRENCY: "string", - REVENUE: "int", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_TRAITS_CITY: "string", - CONTEXT_TRAITS_COUNTRY: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_TRAITS_FIRSTNAME: "string", - CONTEXT_USER_AGENT: "string", - EVENT_TEXT: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - CHANNEL: "string", - EVENT: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - CURRENCY: "USD", - REVENUE: 50, - STACK_HISTORY_ERROR_DETAILS: [ - { - "message": "Cannot set headers after they are sent to the client", - "toString": "[function]" - } + output: { + default: [ + { + metadata: { + table: "identifies", + columns: { + uuid_ts: "datetime", + city: "string", + country: "string", + email: "string", + firstname: "string", + lastname: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + city: "Disney", + country: "UK", + email: "mickey@disney.com", + firstname: "Mickey", + lastname: "Mouse", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "2536eda4-d638-4c93-8014-8ffe3f083214", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.363Z", + timestamp: "2020-01-24T06:29:02.402Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.362Z", + channel: "web" + } + }, + { + metadata: { + table: "users", + columns: { + uuid_ts: "datetime", + city: "string", + country: "string", + email: "string", + firstname: "string", + lastname: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + id: "string", + received_at: "datetime" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + city: "Disney", + country: "UK", + email: "mickey@disney.com", + firstname: "Mickey", + lastname: "Mouse", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + received_at: "2020-01-24T06:29:02.403Z" + } + } ], - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_TRAITS_CITY: "Disney", - CONTEXT_TRAITS_COUNTRY: "USA", - CONTEXT_TRAITS_EMAIL: "mickey@disney.com", - CONTEXT_TRAITS_FIRSTNAME: "Mickey", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - EVENT_TEXT: "button clicked", - ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - SENT_AT: "2020-01-24T06:29:02.364Z", - TIMESTAMP: "2020-01-24T06:29:02.403Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", - CHANNEL: "web", - EVENT: "button_clicked", - } - } - ], - s3_datalake: [ - { - metadata: { - table: "tracks", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - event_text: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - _timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - event: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - event_text: "button clicked", - id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.364Z", - _timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.364Z", - channel: "web", - event: "button_clicked" - } - }, - { - metadata: { - table: "button_clicked", - columns: { - uuid_ts: "datetime", - currency: "string", - revenue: "int", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - event_text: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - _timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - event: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - currency: "USD", - revenue: 50, - stack_history_error_details: [ - { - "message": "Cannot set headers after they are sent to the client", - "toString": "[function]" - } + snowflake: [ + { + metadata: { + table: "IDENTIFIES", + columns: { + UUID_TS: "datetime", + CITY: "string", + COUNTRY: "string", + EMAIL: "string", + FIRSTNAME: "string", + LASTNAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CITY: "Disney", + COUNTRY: "UK", + EMAIL: "mickey@disney.com", + FIRSTNAME: "Mickey", + LASTNAME: "Mouse", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "2536eda4-d638-4c93-8014-8ffe3f083214", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.363Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.362Z", + CHANNEL: "web" + } + }, + { + metadata: { + table: "USERS", + columns: { + UUID_TS: "datetime", + CITY: "string", + COUNTRY: "string", + EMAIL: "string", + FIRSTNAME: "string", + LASTNAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + ID: "string", + RECEIVED_AT: "datetime" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CITY: "Disney", + COUNTRY: "UK", + EMAIL: "mickey@disney.com", + FIRSTNAME: "Mickey", + LASTNAME: "Mouse", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + RECEIVED_AT: "2020-01-24T06:29:02.403Z" + } + } ], - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - event_text: "button clicked", - id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.364Z", - _timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.364Z", - channel: "web", - event: "button_clicked" - } - } - ] - } - }, - identify: { - input: { - destination: { - Config: { - restApiKey: "dummyApiKey", - prefixProperties: true, - useNativeSDK: false - }, - DestinationDefinition: { - DisplayName: "Braze", - ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", - Name: "BRAZE" - }, - Enabled: true, - ID: "1WhcOCGgj9asZu850HvugU2C3Aq", - Name: "Braze", - Transformations: [] - }, - message: { - anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.5" - }, - ip: "0.0.0.0", - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.5" - }, - locale: "en-GB", - os: { - name: "", - version: "" - }, - screen: { - density: 2 - }, - traits: { - city: "Disney", - country: "USA", - email: "mickey@disney.com", - firstname: "Mickey" - }, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" - }, - traits: { - country: "UK", - lastname: "Mouse" - }, - integrations: { - All: true - }, - messageId: "2536eda4-d638-4c93-8014-8ffe3f083214", - originalTimestamp: "2020-01-24T06:29:02.362Z", - receivedAt: "2020-01-24T11:59:02.403+05:30", - request_ip: "[::1]:53708", - sentAt: "2020-01-24T06:29:02.363Z", - timestamp: "2020-01-24T11:59:02.402+05:30", - type: "identify", - userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" - }, - request: { - query: { - whSchemaVersion: "v1" + s3_datalake: [ + { + metadata: { + table: "identifies", + columns: { + uuid_ts: "datetime", + city: "string", + country: "string", + email: "string", + firstname: "string", + lastname: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + _timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + city: "Disney", + country: "UK", + email: "mickey@disney.com", + firstname: "Mickey", + lastname: "Mouse", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "2536eda4-d638-4c93-8014-8ffe3f083214", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.363Z", + _timestamp: "2020-01-24T06:29:02.402Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.362Z", + channel: "web" + } + }, + { + metadata: { + table: "users", + columns: { + uuid_ts: "datetime", + city: "string", + country: "string", + email: "string", + firstname: "string", + lastname: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + id: "string", + received_at: "datetime" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + city: "Disney", + country: "UK", + email: "mickey@disney.com", + firstname: "Mickey", + lastname: "Mouse", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + received_at: "2020-01-24T06:29:02.403Z" + } + } + ] } - } }, - output: { - default: [ - { - metadata: { - table: "identifies", - columns: { - uuid_ts: "datetime", - city: "string", - country: "string", - email: "string", - firstname: "string", - lastname: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string" + page: { + input: { + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + os: { + name: "", + version: "" + }, + screen: { + density: 2 + }, + traits: {}, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true + }, + messageId: "dd266c67-9199-4a52-ba32-f46ddde67312", + originalTimestamp: "2020-01-24T06:29:02.358Z", + properties: { + path: "/tests/html/index2.html", + referrer: "", + search: "", + title: "", + url: "http://localhost/tests/html/index2.html" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.359Z", + timestamp: "2020-01-24T11:59:02.402+05:30", + type: "page", + name: "sample title", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - city: "Disney", - country: "UK", - email: "mickey@disney.com", - firstname: "Mickey", - lastname: "Mouse", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "2536eda4-d638-4c93-8014-8ffe3f083214", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.363Z", - timestamp: "2020-01-24T06:29:02.402Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.362Z", - channel: "web" - } - }, - { - metadata: { - table: "users", - columns: { - uuid_ts: "datetime", - city: "string", - country: "string", - email: "string", - firstname: "string", - lastname: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - id: "string", - received_at: "datetime" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - city: "Disney", - country: "UK", - email: "mickey@disney.com", - firstname: "Mickey", - lastname: "Mouse", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - received_at: "2020-01-24T06:29:02.403Z" - } - } - ], - snowflake: [ - { - metadata: { - table: "IDENTIFIES", - columns: { - UUID_TS: "datetime", - CITY: "string", - COUNTRY: "string", - EMAIL: "string", - FIRSTNAME: "string", - LASTNAME: "string", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_TRAITS_CITY: "string", - CONTEXT_TRAITS_COUNTRY: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_TRAITS_FIRSTNAME: "string", - CONTEXT_USER_AGENT: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - CHANNEL: "string" + destination: { + Config: { + restApiKey: "dummyApiKey", + prefixProperties: true, + useNativeSDK: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + DestinationDefinition: { + DisplayName: "Braze", + ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", + Name: "BRAZE" + }, + Enabled: true, + ID: "1WhcOCGgj9asZu850HvugU2C3Aq", + Name: "Braze", + Transformations: [] }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - CITY: "Disney", - COUNTRY: "UK", - EMAIL: "mickey@disney.com", - FIRSTNAME: "Mickey", - LASTNAME: "Mouse", - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_TRAITS_CITY: "Disney", - CONTEXT_TRAITS_COUNTRY: "USA", - CONTEXT_TRAITS_EMAIL: "mickey@disney.com", - CONTEXT_TRAITS_FIRSTNAME: "Mickey", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - ID: "2536eda4-d638-4c93-8014-8ffe3f083214", - ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - SENT_AT: "2020-01-24T06:29:02.363Z", - TIMESTAMP: "2020-01-24T06:29:02.402Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.362Z", - CHANNEL: "web" - } + request: { + query: { + whSchemaVersion: "v1" + } + } }, - { - metadata: { - table: "USERS", - columns: { - UUID_TS: "datetime", - CITY: "string", - COUNTRY: "string", - EMAIL: "string", - FIRSTNAME: "string", - LASTNAME: "string", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_TRAITS_CITY: "string", - CONTEXT_TRAITS_COUNTRY: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_TRAITS_FIRSTNAME: "string", - CONTEXT_USER_AGENT: "string", - ID: "string", - RECEIVED_AT: "datetime" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - CITY: "Disney", - COUNTRY: "UK", - EMAIL: "mickey@disney.com", - FIRSTNAME: "Mickey", - LASTNAME: "Mouse", - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_TRAITS_CITY: "Disney", - CONTEXT_TRAITS_COUNTRY: "USA", - CONTEXT_TRAITS_EMAIL: "mickey@disney.com", - CONTEXT_TRAITS_FIRSTNAME: "Mickey", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - RECEIVED_AT: "2020-01-24T06:29:02.403Z" - } + output: { + default: [ + { + metadata: { + table: "pages", + columns: { + uuid_ts: "datetime", + path: "string", + url: "string", + name: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_user_agent: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + path: "/tests/html/index2.html", + url: "http://localhost/tests/html/index2.html", + name: "sample title", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "dd266c67-9199-4a52-ba32-f46ddde67312", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.359Z", + timestamp: "2020-01-24T06:29:02.402Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.358Z", + channel: "web" + } + } + ], + snowflake: [ + { + metadata: { + table: "PAGES", + columns: { + UUID_TS: "datetime", + PATH: "string", + URL: "string", + NAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_USER_AGENT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + PATH: "/tests/html/index2.html", + URL: "http://localhost/tests/html/index2.html", + NAME: "sample title", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "dd266c67-9199-4a52-ba32-f46ddde67312", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.359Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.358Z", + CHANNEL: "web" + } + } + ], + s3_datalake: [ + { + metadata: { + table: "pages", + columns: { + uuid_ts: "datetime", + path: "string", + url: "string", + name: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_user_agent: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + _timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + path: "/tests/html/index2.html", + url: "http://localhost/tests/html/index2.html", + name: "sample title", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "dd266c67-9199-4a52-ba32-f46ddde67312", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.359Z", + _timestamp: "2020-01-24T06:29:02.402Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.358Z", + channel: "web" + } + } + ] } - ], - s3_datalake: [ - { - metadata: { - table: "identifies", - columns: { - uuid_ts: "datetime", - city: "string", - country: "string", - email: "string", - firstname: "string", - lastname: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - _timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string" + }, + screen: { + input: { + message: { + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.0" + }, + traits: { + email: "test@rudderstack.com" + }, + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.0" + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + locale: "en-US", + ip: "0.0.0.0", + os: { + name: "", + version: "" + }, + screen: { + density: 2 + } + }, + type: "screen", + messageId: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + originalTimestamp: "2019-10-14T11:15:18.299Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + timestamp: "2020-01-24T11:59:02.402+05:30", + anonymousId: "00000000000000000000000000", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + request_ip: "[::1]:53708", + properties: { + path: "/abc", + referrer: "", + search: "", + title: "", + url: "", + category: "test-category" + }, + integrations: { + All: true + }, + name: "ApplicationLoaded", + sentAt: "2019-10-14T11:15:53.296Z" }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - city: "Disney", - country: "UK", - email: "mickey@disney.com", - firstname: "Mickey", - lastname: "Mouse", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "2536eda4-d638-4c93-8014-8ffe3f083214", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.363Z", - _timestamp: "2020-01-24T06:29:02.402Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.362Z", - channel: "web" - } - }, - { - metadata: { - table: "users", - columns: { - uuid_ts: "datetime", - city: "string", - country: "string", - email: "string", - firstname: "string", - lastname: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - id: "string", - received_at: "datetime" + destination: { + Config: { + apiKey: "12345", + mapToSingleEvent: false, + trackAllPages: false, + trackCategorisedPages: true, + trackNamedPages: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + Enabled: true }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - city: "Disney", - country: "UK", - email: "mickey@disney.com", - firstname: "Mickey", - lastname: "Mouse", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - received_at: "2020-01-24T06:29:02.403Z" - } - } - ] - } - }, - page: { - input: { - message: { - anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.5" - }, - ip: "0.0.0.0", - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.5" - }, - locale: "en-GB", - os: { - name: "", - version: "" - }, - screen: { - density: 2 - }, - traits: {}, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" - }, - integrations: { - All: true - }, - messageId: "dd266c67-9199-4a52-ba32-f46ddde67312", - originalTimestamp: "2020-01-24T06:29:02.358Z", - properties: { - path: "/tests/html/index2.html", - referrer: "", - search: "", - title: "", - url: "http://localhost/tests/html/index2.html" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30", - request_ip: "[::1]:53708", - sentAt: "2020-01-24T06:29:02.359Z", - timestamp: "2020-01-24T11:59:02.402+05:30", - type: "page", - name: "sample title", - userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" - }, - destination: { - Config: { - restApiKey: "dummyApiKey", - prefixProperties: true, - useNativeSDK: false - }, - DestinationDefinition: { - DisplayName: "Braze", - ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", - Name: "BRAZE" + request: { + query: { + whSchemaVersion: "v1" + } + } }, - Enabled: true, - ID: "1WhcOCGgj9asZu850HvugU2C3Aq", - Name: "Braze", - Transformations: [] - }, - request: { - query: { - whSchemaVersion: "v1" + output: { + default: [ + { + metadata: { + table: "screens", + columns: { + uuid_ts: "datetime", + path: "string", + category: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_traits_email: "string", + context_library_name: "string", + context_library_version: "string", + context_user_agent: "string", + context_locale: "string", + context_screen_density: "int", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + original_timestamp: "datetime", + received_at: "datetime", + timestamp: "datetime", + channel: "string", + name: "string" + } + }, + data: { + path: "/abc", + category: "test-category", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.0", + context_traits_email: "test@rudderstack.com", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.0", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + context_locale: "en-US", + context_screen_density: 2, + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + id: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + anonymous_id: "00000000000000000000000000", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2019-10-14T11:15:53.296Z", + original_timestamp: "2019-10-14T11:15:18.299Z", + received_at: "2020-01-24T06:29:02.403Z", + timestamp: "2020-01-24T06:29:02.402Z", + channel: "web", + name: "ApplicationLoaded" + } + } + ], + snowflake: [ + { + metadata: { + table: "SCREENS", + columns: { + UUID_TS: "datetime", + PATH: "string", + CATEGORY: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + TIMESTAMP: "datetime", + CHANNEL: "string", + NAME: "string" + } + }, + data: { + PATH: "/abc", + CATEGORY: "test-category", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.0", + CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.0", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + CONTEXT_LOCALE: "en-US", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + ID: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + ANONYMOUS_ID: "00000000000000000000000000", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2019-10-14T11:15:53.296Z", + ORIGINAL_TIMESTAMP: "2019-10-14T11:15:18.299Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + CHANNEL: "web", + NAME: "ApplicationLoaded" + } + } + ], + s3_datalake: [ + { + metadata: { + table: "screens", + columns: { + uuid_ts: "datetime", + path: "string", + category: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_traits_email: "string", + context_library_name: "string", + context_library_version: "string", + context_user_agent: "string", + context_locale: "string", + context_screen_density: "int", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + original_timestamp: "datetime", + received_at: "datetime", + _timestamp: "datetime", + channel: "string", + name: "string" + } + }, + data: { + path: "/abc", + category: "test-category", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.0", + context_traits_email: "test@rudderstack.com", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.0", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + context_locale: "en-US", + context_screen_density: 2, + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + id: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + anonymous_id: "00000000000000000000000000", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2019-10-14T11:15:53.296Z", + original_timestamp: "2019-10-14T11:15:18.299Z", + received_at: "2020-01-24T06:29:02.403Z", + _timestamp: "2020-01-24T06:29:02.402Z", + channel: "web", + name: "ApplicationLoaded" + } + } + ] } - } }, - output: { - default: [ - { - metadata: { - table: "pages", - columns: { - uuid_ts: "datetime", - path: "string", - url: "string", - name: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_user_agent: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string" + group: { + input: { + message: { + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.0" + }, + traits: { + email: "test@rudderstack.com" + }, + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.0" + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + locale: "en-US", + ip: "0.0.0.0", + os: { + id: "72e528f869711c3d", + manufacturer: "Google", + model: "sdk_gphone_x86", + name: "generic_x86_arm", + token: "some_device_token", + type: "android" + }, + screen: { + density: 2 + } + }, + type: "group", + messageId: "84e26acc-56a5-4835-8233-591137fca468", + originalTimestamp: "2019-10-14T09:03:17.562Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + receivedAt: "2020-01-24T11:59:02.403+05:30", + anonymousId: "00000000000000000000000000", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + groupId: "g1", + integrations: { + All: true + }, + sentAt: "2019-10-14T09:03:22.563Z" }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - path: "/tests/html/index2.html", - url: "http://localhost/tests/html/index2.html", - name: "sample title", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "dd266c67-9199-4a52-ba32-f46ddde67312", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.359Z", - timestamp: "2020-01-24T06:29:02.402Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.358Z", - channel: "web" - } - } - ], - snowflake: [ - { - metadata: { - table: "PAGES", - columns: { - UUID_TS: "datetime", - PATH: "string", - URL: "string", - NAME: "string", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_USER_AGENT: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - CHANNEL: "string" + destination: { + Config: { + apiKey: "12345", + mapToSingleEvent: false, + trackAllPages: true, + trackCategorisedPages: false, + trackNamedPages: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + Enabled: true }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - PATH: "/tests/html/index2.html", - URL: "http://localhost/tests/html/index2.html", - NAME: "sample title", - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - ID: "dd266c67-9199-4a52-ba32-f46ddde67312", - ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - SENT_AT: "2020-01-24T06:29:02.359Z", - TIMESTAMP: "2020-01-24T06:29:02.402Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.358Z", - CHANNEL: "web" - } - } - ], - s3_datalake: [ - { - metadata: { - table: "pages", - columns: { - uuid_ts: "datetime", - path: "string", - url: "string", - name: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_user_agent: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - _timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - path: "/tests/html/index2.html", - url: "http://localhost/tests/html/index2.html", - name: "sample title", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "dd266c67-9199-4a52-ba32-f46ddde67312", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.359Z", - _timestamp: "2020-01-24T06:29:02.402Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.358Z", - channel: "web" - } - } - ] - } - }, - screen: { - input: { - message: { - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.0" - }, - traits: { - email: "test@rudderstack.com" - }, - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.0" - }, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - locale: "en-US", - ip: "0.0.0.0", - os: { - name: "", - version: "" - }, - screen: { - density: 2 - } - }, - type: "screen", - messageId: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", - originalTimestamp: "2019-10-14T11:15:18.299Z", - receivedAt: "2020-01-24T11:59:02.403+05:30", - timestamp: "2020-01-24T11:59:02.402+05:30", - anonymousId: "00000000000000000000000000", - userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - request_ip: "[::1]:53708", - properties: { - path: "/abc", - referrer: "", - search: "", - title: "", - url: "", - category: "test-category" - }, - integrations: { - All: true - }, - name: "ApplicationLoaded", - sentAt: "2019-10-14T11:15:53.296Z" - }, - destination: { - Config: { - apiKey: "12345", - mapToSingleEvent: false, - trackAllPages: false, - trackCategorisedPages: true, - trackNamedPages: false - }, - Enabled: true - }, - request: { - query: { - whSchemaVersion: "v1" - } - } - }, - output: { - default: [ - { - metadata: { - table: "screens", - columns: { - uuid_ts: "datetime", - path: "string", - category: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_traits_email: "string", - context_library_name: "string", - context_library_version: "string", - context_user_agent: "string", - context_locale: "string", - context_screen_density: "int", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - original_timestamp: "datetime", - received_at: "datetime", - timestamp: "datetime", - channel: "string", - name: "string" - } - }, - data: { - path: "/abc", - category: "test-category", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.0", - context_traits_email: "test@rudderstack.com", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.0", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - context_locale: "en-US", - context_screen_density: 2, - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - id: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", - anonymous_id: "00000000000000000000000000", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2019-10-14T11:15:53.296Z", - original_timestamp: "2019-10-14T11:15:18.299Z", - received_at: "2020-01-24T06:29:02.403Z", - timestamp: "2020-01-24T06:29:02.402Z", - channel: "web", - name: "ApplicationLoaded" - } - } - ], - snowflake: [ - { - metadata: { - table: "SCREENS", - columns: { - UUID_TS: "datetime", - PATH: "string", - CATEGORY: "string", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_USER_AGENT: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - TIMESTAMP: "datetime", - CHANNEL: "string", - NAME: "string" - } - }, - data: { - PATH: "/abc", - CATEGORY: "test-category", - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.0", - CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.0", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - CONTEXT_LOCALE: "en-US", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - ID: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", - ANONYMOUS_ID: "00000000000000000000000000", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - SENT_AT: "2019-10-14T11:15:53.296Z", - ORIGINAL_TIMESTAMP: "2019-10-14T11:15:18.299Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - TIMESTAMP: "2020-01-24T06:29:02.402Z", - CHANNEL: "web", - NAME: "ApplicationLoaded" - } - } - ], - s3_datalake: [ - { - metadata: { - table: "screens", - columns: { - uuid_ts: "datetime", - path: "string", - category: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_traits_email: "string", - context_library_name: "string", - context_library_version: "string", - context_user_agent: "string", - context_locale: "string", - context_screen_density: "int", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - original_timestamp: "datetime", - received_at: "datetime", - _timestamp: "datetime", - channel: "string", - name: "string" + request: { + query: { + whSchemaVersion: "v1" + } } - }, - data: { - path: "/abc", - category: "test-category", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.0", - context_traits_email: "test@rudderstack.com", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.0", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - context_locale: "en-US", - context_screen_density: 2, - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - id: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", - anonymous_id: "00000000000000000000000000", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2019-10-14T11:15:53.296Z", - original_timestamp: "2019-10-14T11:15:18.299Z", - received_at: "2020-01-24T06:29:02.403Z", - _timestamp: "2020-01-24T06:29:02.402Z", - channel: "web", - name: "ApplicationLoaded" - } - } - ] - } - }, - group: { - input: { - message: { - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.0" - }, - traits: { - email: "test@rudderstack.com" - }, - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.0" - }, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - locale: "en-US", - ip: "0.0.0.0", - os: { - id: "72e528f869711c3d", - manufacturer: "Google", - model: "sdk_gphone_x86", - name: "generic_x86_arm", - token: "some_device_token", - type: "android" - }, - screen: { - density: 2 - } }, - type: "group", - messageId: "84e26acc-56a5-4835-8233-591137fca468", - originalTimestamp: "2019-10-14T09:03:17.562Z", - timestamp: "2020-01-24T11:59:02.403+05:30", - receivedAt: "2020-01-24T11:59:02.403+05:30", - anonymousId: "00000000000000000000000000", - userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - groupId: "g1", - integrations: { - All: true - }, - sentAt: "2019-10-14T09:03:22.563Z" - }, - destination: { - Config: { - apiKey: "12345", - mapToSingleEvent: false, - trackAllPages: true, - trackCategorisedPages: false, - trackNamedPages: false - }, - Enabled: true - }, - request: { - query: { - whSchemaVersion: "v1" + output: { + default: [ + { + metadata: { + table: "groups", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_traits_email: "string", + context_library_name: "string", + context_library_version: "string", + context_user_agent: "string", + context_locale: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_os_id: "string", + context_os_manufacturer: "string", + context_os_model: "string", + context_os_type: "string", + context_screen_density: "int", + id: "string", + anonymous_id: "string", + user_id: "string", + group_id: "string", + sent_at: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + timestamp: "datetime", + channel: "string" + } + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.0", + context_traits_email: "test@rudderstack.com", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.0", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + context_locale: "en-US", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_os_id: "72e528f869711c3d", + context_os_manufacturer: "Google", + context_os_model: "sdk_gphone_x86", + context_os_name: "", + context_os_token: "", + context_os_type: "android", + context_screen_density: 2, + id: "84e26acc-56a5-4835-8233-591137fca468", + anonymous_id: "00000000000000000000000000", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + group_id: "g1", + sent_at: "2019-10-14T09:03:22.563Z", + original_timestamp: "2019-10-14T09:03:17.562Z", + timestamp: "2020-01-24T11:59:02.403Z", + received_at: "2020-01-24T11:59:02.403Z", + channel: "web" + } + } + ], + snowflake: [ + { + metadata: { + table: "GROUPS", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_LOCALE: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_OS_ID: "string", + CONTEXT_OS_MANUFACTURER: "string", + CONTEXT_OS_MODEL: "string", + CONTEXT_OS_TYPE: "string", + CONTEXT_SCREEN_DENSITY: "int", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + GROUP_ID: "string", + SENT_AT: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + TIMESTAMP: "datetime", + CHANNEL: "string" + } + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.0", + CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.0", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + CONTEXT_LOCALE: "en-US", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_OS_ID: "72e528f869711c3d", + CONTEXT_OS_MANUFACTURER: "Google", + CONTEXT_OS_MODEL: "sdk_gphone_x86", + CONTEXT_OS_NAME: "", + CONTEXT_OS_TOKEN: "", + CONTEXT_OS_TYPE: "android", + CONTEXT_SCREEN_DENSITY: 2, + ID: "84e26acc-56a5-4835-8233-591137fca468", + ANONYMOUS_ID: "00000000000000000000000000", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + GROUP_ID: "g1", + SENT_AT: "2019-10-14T09:03:22.563Z", + ORIGINAL_TIMESTAMP: "2019-10-14T09:03:17.562Z", + TIMESTAMP: "2020-01-24T11:59:02.403Z", + RECEIVED_AT: "2020-01-24T11:59:02.403Z", + CHANNEL: "web" + } + } + ], + s3_datalake: [ + { + metadata: { + table: "groups", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_traits_email: "string", + context_library_name: "string", + context_library_version: "string", + context_user_agent: "string", + context_locale: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_os_id: "string", + context_os_manufacturer: "string", + context_os_model: "string", + context_os_type: "string", + context_screen_density: "int", + id: "string", + anonymous_id: "string", + user_id: "string", + group_id: "string", + sent_at: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + _timestamp: "datetime", + channel: "string" + } + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.0", + context_traits_email: "test@rudderstack.com", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.0", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + context_locale: "en-US", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_os_id: "72e528f869711c3d", + context_os_manufacturer: "Google", + context_os_model: "sdk_gphone_x86", + context_os_name: "", + context_os_token: "", + context_os_type: "android", + context_screen_density: 2, + id: "84e26acc-56a5-4835-8233-591137fca468", + anonymous_id: "00000000000000000000000000", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + group_id: "g1", + sent_at: "2019-10-14T09:03:22.563Z", + original_timestamp: "2019-10-14T09:03:17.562Z", + _timestamp: "2020-01-24T11:59:02.403Z", + received_at: "2020-01-24T11:59:02.403Z", + channel: "web" + } + } + ] } - } }, - output: { - default: [ - { - metadata: { - table: "groups", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_traits_email: "string", - context_library_name: "string", - context_library_version: "string", - context_user_agent: "string", - context_locale: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_os_id: "string", - context_os_manufacturer: "string", - context_os_model: "string", - context_os_type: "string", - context_screen_density: "int", - id: "string", - anonymous_id: "string", - user_id: "string", - group_id: "string", - sent_at: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - timestamp: "datetime", - channel: "string" - } - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.0", - context_traits_email: "test@rudderstack.com", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.0", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - context_locale: "en-US", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_os_id: "72e528f869711c3d", - context_os_manufacturer: "Google", - context_os_model: "sdk_gphone_x86", - context_os_name: "", - context_os_token: "", - context_os_type: "android", - context_screen_density: 2, - id: "84e26acc-56a5-4835-8233-591137fca468", - anonymous_id: "00000000000000000000000000", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - group_id: "g1", - sent_at: "2019-10-14T09:03:22.563Z", - original_timestamp: "2019-10-14T09:03:17.562Z", - timestamp: "2020-01-24T11:59:02.403Z", - received_at: "2020-01-24T11:59:02.403Z", - channel: "web" - } - } - ], - snowflake: [ - { - metadata: { - table: "GROUPS", - columns: { - UUID_TS: "datetime", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_USER_AGENT: "string", - CONTEXT_LOCALE: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_OS_ID: "string", - CONTEXT_OS_MANUFACTURER: "string", - CONTEXT_OS_MODEL: "string", - CONTEXT_OS_TYPE: "string", - CONTEXT_SCREEN_DENSITY: "int", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - GROUP_ID: "string", - SENT_AT: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - TIMESTAMP: "datetime", - CHANNEL: "string" - } - }, - data: { - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.0", - CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.0", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - CONTEXT_LOCALE: "en-US", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_OS_ID: "72e528f869711c3d", - CONTEXT_OS_MANUFACTURER: "Google", - CONTEXT_OS_MODEL: "sdk_gphone_x86", - CONTEXT_OS_NAME: "", - CONTEXT_OS_TOKEN: "", - CONTEXT_OS_TYPE: "android", - CONTEXT_SCREEN_DENSITY: 2, - ID: "84e26acc-56a5-4835-8233-591137fca468", - ANONYMOUS_ID: "00000000000000000000000000", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - GROUP_ID: "g1", - SENT_AT: "2019-10-14T09:03:22.563Z", - ORIGINAL_TIMESTAMP: "2019-10-14T09:03:17.562Z", - TIMESTAMP: "2020-01-24T11:59:02.403Z", - RECEIVED_AT: "2020-01-24T11:59:02.403Z", - CHANNEL: "web" - } - } - ], - s3_datalake: [ - { - metadata: { - table: "groups", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_traits_email: "string", - context_library_name: "string", - context_library_version: "string", - context_user_agent: "string", - context_locale: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_os_id: "string", - context_os_manufacturer: "string", - context_os_model: "string", - context_os_type: "string", - context_screen_density: "int", - id: "string", - anonymous_id: "string", - user_id: "string", - group_id: "string", - sent_at: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - _timestamp: "datetime", - channel: "string" + alias: { + input: { + destination: { + Config: { + apiKey: "dummyApiKey", + prefixProperties: true, + useNativeSDK: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + DestinationDefinition: { + DisplayName: "Kiss Metrics", + ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", + Name: "KISSMETRICS" + }, + Enabled: true, + ID: "1WhcOCGgj9asZu850HvugU2C3Aq", + Name: "Kiss Metrics", + Transformations: [] + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + os: { + name: "", + version: "" + }, + screen: { + density: 2 + }, + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true + }, + messageId: "79313729-7fe5-4204-963a-dc46f4205e4e", + originalTimestamp: "2020-01-24T06:29:02.366Z", + previousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.366Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "alias", + userId: "1234abc" + }, + request: { + query: { + whSchemaVersion: "v1" + } } - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.0", - context_traits_email: "test@rudderstack.com", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.0", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - context_locale: "en-US", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_os_id: "72e528f869711c3d", - context_os_manufacturer: "Google", - context_os_model: "sdk_gphone_x86", - context_os_name: "", - context_os_token: "", - context_os_type: "android", - context_screen_density: 2, - id: "84e26acc-56a5-4835-8233-591137fca468", - anonymous_id: "00000000000000000000000000", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - group_id: "g1", - sent_at: "2019-10-14T09:03:22.563Z", - original_timestamp: "2019-10-14T09:03:17.562Z", - _timestamp: "2020-01-24T11:59:02.403Z", - received_at: "2020-01-24T11:59:02.403Z", - channel: "web" - } - } - ] - } - }, - alias: { - input: { - destination: { - Config: { - apiKey: "dummyApiKey", - prefixProperties: true, - useNativeSDK: false }, - DestinationDefinition: { - DisplayName: "Kiss Metrics", - ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", - Name: "KISSMETRICS" - }, - Enabled: true, - ID: "1WhcOCGgj9asZu850HvugU2C3Aq", - Name: "Kiss Metrics", - Transformations: [] - }, - message: { - anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.5" - }, - ip: "0.0.0.0", - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.5" - }, - locale: "en-GB", - os: { - name: "", - version: "" - }, - screen: { - density: 2 - }, - traits: { - city: "Disney", - country: "USA", - email: "mickey@disney.com", - firstname: "Mickey" - }, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" - }, - integrations: { - All: true - }, - messageId: "79313729-7fe5-4204-963a-dc46f4205e4e", - originalTimestamp: "2020-01-24T06:29:02.366Z", - previousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - receivedAt: "2020-01-24T11:59:02.403+05:30", - request_ip: "[::1]:53708", - sentAt: "2020-01-24T06:29:02.366Z", - timestamp: "2020-01-24T11:59:02.403+05:30", - type: "alias", - userId: "1234abc" - }, - request: { - query: { - whSchemaVersion: "v1" + output: { + default: [ + { + metadata: { + table: "aliases", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + previous_id: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "79313729-7fe5-4204-963a-dc46f4205e4e", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "1234abc", + sent_at: "2020-01-24T06:29:02.366Z", + timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.366Z", + channel: "web", + previous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" + } + } + ], + snowflake: [ + { + metadata: { + table: "ALIASES", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + PREVIOUS_ID: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "79313729-7fe5-4204-963a-dc46f4205e4e", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "1234abc", + SENT_AT: "2020-01-24T06:29:02.366Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.366Z", + CHANNEL: "web", + PREVIOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" + } + } + ], + s3_datalake: [ + { + metadata: { + table: "aliases", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + _timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + previous_id: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "79313729-7fe5-4204-963a-dc46f4205e4e", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "1234abc", + sent_at: "2020-01-24T06:29:02.366Z", + _timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.366Z", + channel: "web", + previous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" + } + } + ] } - } }, - output: { - default: [ - { - metadata: { - table: "aliases", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - previous_id: "string" + extract: { + input: { + destination: { + Config: { + restApiKey: "dummyApiKey", + prefixProperties: true, + useNativeSDK: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + DestinationDefinition: { + DisplayName: "Braze", + ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", + Name: "BRAZE" + }, + Enabled: true, + ID: "1WhcOCGgj9asZu850HvugU2C3Aq", + Name: "Braze", + Transformations: [] }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "79313729-7fe5-4204-963a-dc46f4205e4e", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "1234abc", - sent_at: "2020-01-24T06:29:02.366Z", - timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.366Z", - channel: "web", - previous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" - } - } - ], - snowflake: [ - { - metadata: { - table: "ALIASES", - columns: { - UUID_TS: "datetime", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_TRAITS_CITY: "string", - CONTEXT_TRAITS_COUNTRY: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_TRAITS_FIRSTNAME: "string", - CONTEXT_USER_AGENT: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - CHANNEL: "string", - PREVIOUS_ID: "string" + message: { + channel: "sources", + context: { + sources: { + job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", + job_run_id: "cfd6705nsevh5p2l77ag", + task_run_id: "cfd6705nsevh5p2l77b0", + version: "version" + } + }, + properties: { + string_property: "value1", + date_property: "2023-02-01T07:53:31.430Z", + boolean_property: true, + date_property2: "2023-01-31T07:39:10.002Z", + userId: "dummy-user-id-inside-properties" + }, + event: "extract_event", + recordId: "some-uuid", + originalTimestamp: "2020-01-24T06:29:02.364Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "extract", + userId: "dummy-user-id-outside-properties" }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_TRAITS_CITY: "Disney", - CONTEXT_TRAITS_COUNTRY: "USA", - CONTEXT_TRAITS_EMAIL: "mickey@disney.com", - CONTEXT_TRAITS_FIRSTNAME: "Mickey", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - ID: "79313729-7fe5-4204-963a-dc46f4205e4e", - ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - USER_ID: "1234abc", - SENT_AT: "2020-01-24T06:29:02.366Z", - TIMESTAMP: "2020-01-24T06:29:02.403Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.366Z", - CHANNEL: "web", - PREVIOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" - } - } - ], - s3_datalake: [ - { - metadata: { - table: "aliases", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - _timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - previous_id: "string" + request: { + query: { + whSchemaVersion: "v1" + } }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "79313729-7fe5-4204-963a-dc46f4205e4e", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "1234abc", - sent_at: "2020-01-24T06:29:02.366Z", - _timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.366Z", - channel: "web", - previous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" - } - } - ] - } - }, - extract: { - input: { - destination: { - Config: { - restApiKey: "dummyApiKey", - prefixProperties: true, - useNativeSDK: false - }, - DestinationDefinition: { - DisplayName: "Braze", - ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", - Name: "BRAZE" - }, - Enabled: true, - ID: "1WhcOCGgj9asZu850HvugU2C3Aq", - Name: "Braze", - Transformations: [] - }, - message: { - channel: "sources", - context: { - sources: { - job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", - job_run_id: "cfd6705nsevh5p2l77ag", - task_run_id: "cfd6705nsevh5p2l77b0", - version: "version" - } - }, - properties: { - string_property: "value1", - date_property: "2023-02-01T07:53:31.430Z", - boolean_property: true, - date_property2: "2023-01-31T07:39:10.002Z", - userId: "dummy-user-id-inside-properties" + metadata: { + sourceCategory: "singer-protocol", + } }, - event: "extract_event", - recordId: "some-uuid", - originalTimestamp: "2020-01-24T06:29:02.364Z", - receivedAt: "2020-01-24T11:59:02.403+05:30", - request_ip: "[::1]:53708", - sentAt: "2020-01-24T06:29:02.364Z", - timestamp: "2020-01-24T11:59:02.403+05:30", - type: "extract", - userId: "dummy-user-id-outside-properties" - }, - request: { - query: { - whSchemaVersion: "v1" + output: { + default: [ + { + metadata: { + table: "extract_event", + columns: { + date_property: "datetime", + date_property_2: "datetime", + boolean_property: "boolean", + context_sources_job_id: "string", + context_sources_job_run_id: "string", + context_sources_task_run_id: "string", + context_sources_version: "string", + event: "string", + id: "string", + user_id: "string", + received_at: "datetime", + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + date_property: "2023-02-01T07:53:31.430Z", + date_property_2: "2023-01-31T07:39:10.002Z", + context_sources_job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", + context_sources_job_run_id: "cfd6705nsevh5p2l77ag", + context_sources_task_run_id: "cfd6705nsevh5p2l77b0", + context_sources_version: "version", + id: "some-uuid", + user_id: "dummy-user-id-inside-properties", + received_at: "2020-01-24T06:29:02.403Z", + event: "extract_event", + boolean_property: true + } + } + ], + snowflake: [ + { + metadata: { + table: "EXTRACT_EVENT", + columns: { + DATE_PROPERTY: "datetime", + DATE_PROPERTY_2: "datetime", + CONTEXT_SOURCES_JOB_ID: "string", + CONTEXT_SOURCES_JOB_RUN_ID: "string", + CONTEXT_SOURCES_TASK_RUN_ID: "string", + CONTEXT_SOURCES_VERSION: "string", + EVENT: "string", + ID: "string", + USER_ID: "string", + RECEIVED_AT: "datetime", + BOOLEAN_PROPERTY: "boolean", + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + DATE_PROPERTY: "2023-02-01T07:53:31.430Z", + DATE_PROPERTY_2: "2023-01-31T07:39:10.002Z", + CONTEXT_SOURCES_JOB_ID: "2JABSy1nq89H7xeJimBL2pCtOOp", + CONTEXT_SOURCES_JOB_RUN_ID: "cfd6705nsevh5p2l77ag", + CONTEXT_SOURCES_TASK_RUN_ID: "cfd6705nsevh5p2l77b0", + CONTEXT_SOURCES_VERSION: "version", + ID: "some-uuid", + USER_ID: "dummy-user-id-inside-properties", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + EVENT: "extract_event", + BOOLEAN_PROPERTY: true, + } + } + ], + s3_datalake: [ + { + metadata: { + table: "extract_event", + columns: { + date_property: "datetime", + date_property_2: "datetime", + context_sources_job_id: "string", + context_sources_job_run_id: "string", + context_sources_task_run_id: "string", + context_sources_version: "string", + event: "string", + id: "string", + user_id: "string", + received_at: "datetime", + boolean_property: "boolean", + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + date_property: "2023-02-01T07:53:31.430Z", + date_property_2: "2023-01-31T07:39:10.002Z", + context_sources_job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", + context_sources_job_run_id: "cfd6705nsevh5p2l77ag", + context_sources_task_run_id: "cfd6705nsevh5p2l77b0", + context_sources_version: "version", + id: "some-uuid", + event: "extract_event", + boolean_property: true, + received_at: "2020-01-24T06:29:02.403Z", + user_id: "dummy-user-id-inside-properties", + } + } + ] } - }, - metadata: { - sourceCategory: "singer-protocol", - } }, - output: { - default: [ - { - metadata: { - table: "extract_event", - columns: { - date_property: "datetime", - date_property_2: "datetime", - boolean_property: "boolean", - context_sources_job_id: "string", - context_sources_job_run_id: "string", - context_sources_task_run_id: "string", - context_sources_version: "string", - event: "string", - id: "string", - user_id: "string", - received_at: "datetime", - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - date_property: "2023-02-01T07:53:31.430Z", - date_property_2: "2023-01-31T07:39:10.002Z", - context_sources_job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", - context_sources_job_run_id: "cfd6705nsevh5p2l77ag", - context_sources_task_run_id: "cfd6705nsevh5p2l77b0", - context_sources_version: "version", - id: "some-uuid", - user_id: "dummy-user-id-inside-properties", - received_at: "2020-01-24T06:29:02.403Z", - event: "extract_event", - boolean_property: true - } - } - ], - snowflake: [ - { - metadata: { - table: "EXTRACT_EVENT", - columns: { - DATE_PROPERTY: "datetime", - DATE_PROPERTY_2: "datetime", - CONTEXT_SOURCES_JOB_ID: "string", - CONTEXT_SOURCES_JOB_RUN_ID: "string", - CONTEXT_SOURCES_TASK_RUN_ID: "string", - CONTEXT_SOURCES_VERSION: "string", - EVENT: "string", - ID: "string", - USER_ID: "string", - RECEIVED_AT: "datetime", - BOOLEAN_PROPERTY: "boolean", - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - DATE_PROPERTY: "2023-02-01T07:53:31.430Z", - DATE_PROPERTY_2: "2023-01-31T07:39:10.002Z", - CONTEXT_SOURCES_JOB_ID: "2JABSy1nq89H7xeJimBL2pCtOOp", - CONTEXT_SOURCES_JOB_RUN_ID: "cfd6705nsevh5p2l77ag", - CONTEXT_SOURCES_TASK_RUN_ID: "cfd6705nsevh5p2l77b0", - CONTEXT_SOURCES_VERSION: "version", - ID: "some-uuid", - USER_ID: "dummy-user-id-inside-properties", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - EVENT: "extract_event", - BOOLEAN_PROPERTY: true, - } - } - ], - s3_datalake: [ - { - metadata: { - table: "extract_event", - columns: { - date_property: "datetime", - date_property_2: "datetime", - context_sources_job_id: "string", - context_sources_job_run_id: "string", - context_sources_task_run_id: "string", - context_sources_version: "string", - event: "string", - id: "string", - user_id: "string", - received_at: "datetime", - boolean_property: "boolean", - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - date_property: "2023-02-01T07:53:31.430Z", - date_property_2: "2023-01-31T07:39:10.002Z", - context_sources_job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", - context_sources_job_run_id: "cfd6705nsevh5p2l77ag", - context_sources_task_run_id: "cfd6705nsevh5p2l77b0", - context_sources_version: "version", - id: "some-uuid", - event: "extract_event", - boolean_property: true, - received_at: "2020-01-24T06:29:02.403Z", - user_id: "dummy-user-id-inside-properties", - } - } - ] - } - }, }; function input(eventType) { - return _.cloneDeep(sampleEvents[eventType].input); + return _.cloneDeep(sampleEvents[eventType].input); } function output(eventType, provider) { - if (provider === "snowflake") { - return _.cloneDeep(sampleEvents[eventType].output.snowflake); - } - if (provider === "s3_datalake") { - return _.cloneDeep(sampleEvents[eventType].output.s3_datalake); - } - return _.cloneDeep(sampleEvents[eventType].output.default); + if (provider === "snowflake") { + return _.cloneDeep(sampleEvents[eventType].output.snowflake); + } + if (provider === "s3_datalake") { + return _.cloneDeep(sampleEvents[eventType].output.s3_datalake); + } + return _.cloneDeep(sampleEvents[eventType].output.default); } -module.exports = { input, output }; +module.exports = {input, output}; diff --git a/test/__tests__/data/warehouse/integration_options_events.js b/test/__tests__/data/warehouse/integration_options_events.js index 05a2d51abd..0e1ea10af8 100644 --- a/test/__tests__/data/warehouse/integration_options_events.js +++ b/test/__tests__/data/warehouse/integration_options_events.js @@ -20,7 +20,10 @@ const sampleEvents = { input: { destination: { Config: { - jsonPaths: " testMap.nestedMap, testArray" + jsonPaths: " testMap.nestedMap, testArray", + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true } }, message: { @@ -731,7 +734,11 @@ const sampleEvents = { users: { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { type: "identify", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js index ee748a1a2b..93aea31dd9 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js index 906c3ada23..ddedd87282 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js index 13b243f3a7..219b625707 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js index 47b7f1209a..6abe5b71bd 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { type: "identify", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js index 0aac8a3c23..dd81123757 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js index bad325c908..6bf44e5ba5 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js index 5070284e99..f117894e77 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js index 91f19c5c77..8145d6a3bb 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js index 7a36e4787e..eb27b44a1f 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js index f84f9d33ed..7eb3aba247 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js index 3d6164b430..548ede9614 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { type: "identify", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js index 136f355b21..81008448dc 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js index b11b311ebb..398e013070 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js index 7ed95685ff..04005ff434 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js @@ -1,7 +1,11 @@ module.exports = { input: { destination: { - Config: {} + Config: { + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + } }, message: { anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 2c89120686..d1fc833fe0 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -1,1236 +1,1890 @@ const _ = require("lodash"); -const util = require("util"); - -const { input, output } = require(`./data/warehouse/events.js`); -const { - opInput, - opOutput -} = require(`./data/warehouse/integration_options_events.js`); -const { names } = require(`./data/warehouse/names.js`); +const {input, output} = require(`./data/warehouse/events.js`); +const {opInput, opOutput} = require(`./data/warehouse/integration_options_events.js`); +const {names} = require(`./data/warehouse/names.js`); const destConfig = require(`./data/warehouse/dest_config_scenarios.js`); -const { - largeNoOfColumnsevent -} = require(`./data/warehouse/event_columns_length`); -const { rudderProperties } = require(`./data/warehouse/props.js`); +const {largeNoOfColumnsevent} = require(`./data/warehouse/event_columns_length`); +const {rudderProperties} = require(`./data/warehouse/props.js`); const reservedANSIKeywordsMap = require("../../src/warehouse/config/ReservedKeywords.json"); -const { - fullEventColumnTypeByProvider -} = require("../../src/warehouse/index.js"); - -const { - validTimestamp -} = require("../../src/warehouse/util.js"); +const {fullEventColumnTypeByProvider} = require("../../src/warehouse/index.js"); +const {validTimestamp} = require("../../src/warehouse/util.js"); +const {transformTableName, transformColumnName} = require("../../src/warehouse/v1/util"); const {isBlank} = require("../../src/warehouse/config/helpers.js"); const version = "v0"; const integrations = [ - "rs", - "bq", - "postgres", - "clickhouse", - "snowflake", - "mssql", - "azure_synapse", - "deltalake", - "azure_datalake", - "s3_datalake", - "gcs_datalake", + "rs", + "bq", + "postgres", + "clickhouse", + "snowflake", + "mssql", + "azure_synapse", + "deltalake", + "azure_datalake", + "s3_datalake", + "gcs_datalake", ]; const transformers = integrations.map(integration => - require(`../../src/${version}/destinations/${integration}/transform`) + require(`../../src/${version}/destinations/${integration}/transform`) ); const eventTypes = ["track", "identify", "page", "screen", "group", "alias"]; // get key of user set properties in the event // eg. for identify call, user sets the custom properties inside traits const propsKeyMap = { - track: "properties", - page: "properties", - screen: "properties", - - identify: "traits", - group: "traits", - alias: "traits" + track: "properties", + page: "properties", + screen: "properties", + identify: "traits", + group: "traits", + alias: "traits" }; - const integrationCasedString = (integration, str) => { - if (integration === "snowflake") { - return str.toUpperCase(); - } - return str; + if (integration === "snowflake") { + return str.toUpperCase(); + } + return str; }; describe("event types", () => { - describe("track", () => { - it("should generate two events for every track call", () => { - const i = input("track"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("track", integrations[index])); - }); - }); - }); - - describe("identify", () => { - it("should generate two events for every identify call", () => { - const i = input("identify"); - // also verfies priority order between traits and context.traits - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("identify", integrations[index])); - }); - }); - }); - - describe("page", () => { - it("should generate one event for every page call", () => { - const i = input("page"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("page", integrations[index])); - }); + describe("track", () => { + it("should generate two events for every track call", () => { + const i = input("track"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("track", integrations[index])); + }); + }); }); - it("should take name from properties if top-level name is missing", () => { - let i = input("page"); - i.message.properties.name = i.message.name; - delete i.message.name; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("page", integrations[index])); - }); + + describe("identify", () => { + it("should generate two events for every identify call", () => { + const i = input("identify"); + // also verfies priority order between traits and context.traits + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("identify", integrations[index])); + }); + }); }); - }); - - describe("screen", () => { - it("should generate one event for every screen call", () => { - const i = input("screen"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integrations[index])); - }); + + describe("page", () => { + it("should generate one event for every page call", () => { + const i = input("page"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("page", integrations[index])); + }); + }); + it("should take name from properties if top-level name is missing", () => { + let i = input("page"); + i.message.properties.name = i.message.name; + delete i.message.name; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("page", integrations[index])); + }); + }); }); - it("should take name from properties if top-level name is missing", () => { - let i = input("screen"); - i.message.properties.name = i.message.name; - delete i.message.name; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integrations[index])); - }); + + describe("screen", () => { + it("should generate one event for every screen call", () => { + const i = input("screen"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("screen", integrations[index])); + }); + }); + it("should take name from properties if top-level name is missing", () => { + let i = input("screen"); + i.message.properties.name = i.message.name; + delete i.message.name; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("screen", integrations[index])); + }); + }); }); - }); - - describe("alias", () => { - it("should generate one event for every alias call", () => { - const i = input("alias"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("alias", integrations[index])); - }); + + describe("alias", () => { + it("should generate one event for every alias call", () => { + const i = input("alias"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("alias", integrations[index])); + }); + }); }); - }); - - describe("extract", () => { - it("should generate one event for every extract call", () => { - const i = input("extract"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("extract", integrations[index])); - }); + + describe("extract", () => { + it("should generate one event for every extract call", () => { + const i = input("extract"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("extract", integrations[index])); + }); + }); }); - }); }); describe("column & table names", () => { - it("should handle special character, spacing, snake case etc.", () => { - let i = input("track"); - - i.message.properties = Object.assign( - i.message.properties, - names.input.properties - ); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - - const provider = - integrations[index] === "snowflake" ? "snowflake" : "default"; - - expect(received[1].metadata.columns).toMatchObject( - names.output.columns[provider] - ); - expect(received[1].data).toMatchObject(names.output.data[provider]); - - for (let tableName in names.input.properties) { - i.message.event = tableName; - let out = transformer.process(i); - expect(out[1].metadata.table).toEqual( - names.output.namesMap[provider][tableName] + it("should handle special character, spacing, snake case etc.", () => { + let i = input("track"); + + i.message.properties = Object.assign( + i.message.properties, + names.input.properties ); - } + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + + const provider = + integrations[index] === "snowflake" ? "snowflake" : "default"; + + expect(received[1].metadata.columns).toMatchObject( + names.output.columns[provider] + ); + expect(received[1].data).toMatchObject(names.output.data[provider]); + + for (let tableName in names.input.properties) { + i.message.event = tableName; + let out = transformer.process(i); + expect(out[1].metadata.table).toEqual( + names.output.namesMap[provider][tableName] + ); + } + }); }); - }); - it("should trim column names in postgres", () => { - let i = input("track"); - names.input.properties[ - "a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5" - ] = "70 letter identifier"; - names.input.properties[ - "xa1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5abcdefghi" - ] = "500 letter identifier"; - i.message.properties = Object.assign( - i.message.properties, - names.input.properties - ); - i.message.event = - "a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5"; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - if (integrations[index] === "postgres") { - expect(received[1].metadata).toHaveProperty( - "table", - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1" - ); - expect(received[1].metadata.columns).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1", - "string" - ); - expect(received[1].data).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1", - "70 letter identifier" - ); - expect(received[1].metadata.columns).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_", - "string" - ); - expect(received[1].data).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_", - "500 letter identifier" - ); - //KEY should be trimmed to 63 - return; - } - if (integrations[index] === "snowflake") { - expect(received[1].metadata).toHaveProperty( - "table", - "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2" - ); - expect(received[1].metadata.columns).toHaveProperty( - "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2", - "string" - ); - expect(received[1].data).toHaveProperty( - "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2", - "70 letter identifier" - ); - expect(received[1].metadata.columns).toHaveProperty( - "XA_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_", - "string" - ); - expect(received[1].data).toHaveProperty( - "XA_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_", - "500 letter identifier" - ); - return; - } - if (integrations[index] === "s3_datalake" || integrations[index] === "gcs_datalake" || integrations[index] === "azure_datalake") { - expect(received[1].metadata).toHaveProperty( - "table", - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5" - ); - expect(received[1].metadata.columns).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5", - "string" - ); - expect(received[1].data).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5", - "70 letter identifier" - ); - expect(received[1].metadata.columns).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_abcdefghi", - "string" - ); - expect(received[1].data).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_abcdefghi", - "500 letter identifier" + + it("should trim column names in postgres", () => { + let i = input("track"); + names.input.properties[ + "a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5" + ] = "70 letter identifier"; + names.input.properties[ + "xa1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5abcdefghi" + ] = "500 letter identifier"; + i.message.properties = Object.assign( + i.message.properties, + names.input.properties ); - return; - } - expect(received[1].metadata).toHaveProperty( - "table", - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2" - ); - expect(received[1].metadata.columns).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2", - "string" - ); - expect(received[1].data).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2", - "70 letter identifier" - ); - expect(received[1].metadata.columns).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_", - "string" - ); - expect(received[1].data).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_", - "500 letter identifier" - ); - //KEY should be trimmed to 127 + i.message.event = + "a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5"; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + if (integrations[index] === "postgres") { + expect(received[1].metadata).toHaveProperty( + "table", + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1" + ); + expect(received[1].metadata.columns).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1", + "string" + ); + expect(received[1].data).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1", + "70 letter identifier" + ); + expect(received[1].metadata.columns).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_", + "string" + ); + expect(received[1].data).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_", + "500 letter identifier" + ); + //KEY should be trimmed to 63 + return; + } + if (integrations[index] === "snowflake") { + expect(received[1].metadata).toHaveProperty( + "table", + "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2" + ); + expect(received[1].metadata.columns).toHaveProperty( + "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2", + "string" + ); + expect(received[1].data).toHaveProperty( + "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2", + "70 letter identifier" + ); + expect(received[1].metadata.columns).toHaveProperty( + "XA_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_", + "string" + ); + expect(received[1].data).toHaveProperty( + "XA_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_", + "500 letter identifier" + ); + return; + } + if (integrations[index] === "s3_datalake" || integrations[index] === "gcs_datalake" || integrations[index] === "azure_datalake") { + expect(received[1].metadata).toHaveProperty( + "table", + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5" + ); + expect(received[1].metadata.columns).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5", + "string" + ); + expect(received[1].data).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5", + "70 letter identifier" + ); + expect(received[1].metadata.columns).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_abcdefghi", + "string" + ); + expect(received[1].data).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_abcdefghi", + "500 letter identifier" + ); + return; + } + expect(received[1].metadata).toHaveProperty( + "table", + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2" + ); + expect(received[1].metadata.columns).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2", + "string" + ); + expect(received[1].data).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2", + "70 letter identifier" + ); + expect(received[1].metadata.columns).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_", + "string" + ); + expect(received[1].data).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_", + "500 letter identifier" + ); + //KEY should be trimmed to 127 + }); }); - }); }); // tests case where properties set by user match the columns set by rudder(id, recevied etc) // https://docs.rudderstack.com/data-warehouse-integrations/warehouse-schemas#standard-rudderstack-properties describe("conflict between rudder set props and user set props", () => { - it("should override user set props with rudder prop", () => { - eventTypes.forEach(evType => { - const i = input(evType); - - const propsKey = propsKeyMap[evType]; - const rudderProps = _.compact( - rudderProperties.default.concat(rudderProperties[evType]) - ); - - rudderProps.forEach(prop => { - // _.set creates inner object if not present - _.set(i.message, `${propsKey}.${prop}`, "test prop"); - }); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - rudderProps.forEach(prop => { - if (received[0].data[prop] === "test prop") { - } - expect(received[0].data[prop]).not.toBe("test prop"); + it("should override user set props with rudder prop", () => { + eventTypes.forEach(evType => { + const i = input(evType); + + const propsKey = propsKeyMap[evType]; + const rudderProps = _.compact( + rudderProperties.default.concat(rudderProperties[evType]) + ); + + rudderProps.forEach(prop => { + // _.set creates inner object if not present + _.set(i.message, `${propsKey}.${prop}`, "test prop"); + }); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + rudderProps.forEach(prop => { + if (received[0].data[prop] === "test prop") { + } + expect(received[0].data[prop]).not.toBe("test prop"); + }); + }); }); - }); }); - }); - - it("should set data type of rudder prop and not user set prop", () => { - eventTypes.forEach(evType => { - let i = input(evType); - - const propsKey = propsKeyMap[evType]; - transformers.forEach((transformer, index) => { - let sampleRudderPropKey = "id"; - if (integrations[index] === "snowflake") { - sampleRudderPropKey = "ID"; - } - _.set(i.message, `${propsKey}.${sampleRudderPropKey}`, true); - const received = transformer.process(i); - expect(received[0].metadata.columns[sampleRudderPropKey]).not.toBe( - "bool" - ); - expect(received[0].metadata.columns[sampleRudderPropKey]).toBe( - "string" - ); - }); + it("should set data type of rudder prop and not user set prop", () => { + eventTypes.forEach(evType => { + let i = input(evType); + + const propsKey = propsKeyMap[evType]; + transformers.forEach((transformer, index) => { + let sampleRudderPropKey = "id"; + if (integrations[index] === "snowflake") { + sampleRudderPropKey = "ID"; + } + + _.set(i.message, `${propsKey}.${sampleRudderPropKey}`, true); + const received = transformer.process(i); + expect(received[0].metadata.columns[sampleRudderPropKey]).not.toBe( + "bool" + ); + expect(received[0].metadata.columns[sampleRudderPropKey]).toBe( + "string" + ); + }); + }); }); - }); }); describe("handle reserved words", () => { - const OLD_ENV = process.env; - beforeEach(() => { - process.env = { ...OLD_ENV }; - process.env.WH_MAX_COLUMNS_IN_EVENT = 600; - jest.resetModules(); - }); - afterAll(() => { - process.env = OLD_ENV; // restore old env - }); - it("prepend underscore", () => { - // re-import transformer modules so that new env values are used - const transformers = integrations.map(integration => - require(`../../src/${version}/destinations/${integration}/transform`) - ); - eventTypes.forEach(evType => { - let i = input(evType); - - const propsKey = propsKeyMap[evType]; - transformers.forEach((transformer, index) => { - const reserverdKeywordsMap = - reservedANSIKeywordsMap[integrations[index].toUpperCase()]; - - i.message[propsKey] = Object.assign( - i.message[propsKey] || {}, - reserverdKeywordsMap + const OLD_ENV = process.env; + beforeEach(() => { + process.env = {...OLD_ENV}; + process.env.WH_MAX_COLUMNS_IN_EVENT = 600; + jest.resetModules(); + }); + afterAll(() => { + process.env = OLD_ENV; // restore old env + }); + it("prepend underscore", () => { + // re-import transformer modules so that new env values are used + const transformers = integrations.map(integration => + require(`../../src/${version}/destinations/${integration}/transform`) ); - - const received = transformer.process(i); - - const out = - evType === "track" || evType === "identify" - ? received[1] - : received[0]; - - Object.keys(reserverdKeywordsMap).forEach(k => { - expect(out.metadata.columns).not.toHaveProperty(k.toLowerCase()); - expect(out.metadata.columns).not.toHaveProperty(k.toUpperCase()); - let snakeCasedKey = _.snakeCase(k).toUpperCase(); - if (k === snakeCasedKey) { - k = `_${k}`; - } else { - k = snakeCasedKey; - } - if (integrations[index] === "snowflake") { - expect(out.metadata.columns).toHaveProperty(k); - } else { - expect(out.metadata.columns).toHaveProperty(k.toLowerCase()); - } + eventTypes.forEach(evType => { + let i = input(evType); + + const propsKey = propsKeyMap[evType]; + transformers.forEach((transformer, index) => { + const reserverdKeywordsMap = + reservedANSIKeywordsMap[integrations[index].toUpperCase()]; + + i.message[propsKey] = Object.assign( + i.message[propsKey] || {}, + reserverdKeywordsMap + ); + + const received = transformer.process(i); + + const out = + evType === "track" || evType === "identify" + ? received[1] + : received[0]; + + Object.keys(reserverdKeywordsMap).forEach(k => { + expect(out.metadata.columns).not.toHaveProperty(k.toLowerCase()); + expect(out.metadata.columns).not.toHaveProperty(k.toUpperCase()); + let snakeCasedKey = _.snakeCase(k).toUpperCase(); + if (k === snakeCasedKey) { + k = `_${k}`; + } else { + k = snakeCasedKey; + } + if (integrations[index] === "snowflake") { + expect(out.metadata.columns).toHaveProperty(k); + } else { + expect(out.metadata.columns).toHaveProperty(k.toLowerCase()); + } + }); + }); }); - }); }); - }); }); describe("null/empty values", () => { - it("should skip setting null/empty value fields", () => { - eventTypes.forEach(evType => { - let i = input(evType); - - const propsKey = propsKeyMap[evType]; - const emptyValsMap = { - empty_val_1: "", - empty_val_2: [], - empty_val_3: null, - empty_val_4: undefined - }; - const emptyValKeys = Object.keys(emptyValsMap).concat( - Object.keys(emptyValsMap).map(k => k.toUpperCase()) - ); - - i.message[propsKey] = Object.assign( - i.message[propsKey] || {}, - emptyValsMap - ); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - emptyValKeys.forEach(k => { - expect(received[0].metadata.columns).not.toHaveProperty(k); - expect(received[0].data).not.toHaveProperty(k); - if (received[1]) { - expect(received[1].metadata.columns).not.toHaveProperty(k); - expect(received[1].data).not.toHaveProperty(k); - } + it("should skip setting null/empty value fields", () => { + eventTypes.forEach(evType => { + let i = input(evType); + + const propsKey = propsKeyMap[evType]; + const emptyValsMap = { + empty_val_1: "", + empty_val_2: [], + empty_val_3: null, + empty_val_4: undefined + }; + const emptyValKeys = Object.keys(emptyValsMap).concat( + Object.keys(emptyValsMap).map(k => k.toUpperCase()) + ); + + i.message[propsKey] = Object.assign( + i.message[propsKey] || {}, + emptyValsMap + ); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + emptyValKeys.forEach(k => { + expect(received[0].metadata.columns).not.toHaveProperty(k); + expect(received[0].data).not.toHaveProperty(k); + if (received[1]) { + expect(received[1].metadata.columns).not.toHaveProperty(k); + expect(received[1].data).not.toHaveProperty(k); + } + }); + }); }); - }); }); - }); }); describe("invalid context", () => { - it("should skip setting context fields if context is not an object", () => { - eventTypes.forEach(evType => { - let i = input(evType); - i.message.context = "{{invalid object}}"; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - let columns = Object.keys(received[0].metadata.columns); - if (received[1]) { - columns = columns.concat(Object.keys(received[1].metadata.columns)); - } - columns.forEach(c => { - expect(c).not.toMatch(/^context_\d/g); + it("should skip setting context fields if context is not an object", () => { + eventTypes.forEach(evType => { + let i = input(evType); + i.message.context = "{{invalid object}}"; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + let columns = Object.keys(received[0].metadata.columns); + if (received[1]) { + columns = columns.concat(Object.keys(received[1].metadata.columns)); + } + columns.forEach(c => { + expect(c).not.toMatch(/^context_\d/g); + }); + let data = Object.keys(received[0].data); + if (received[1]) { + data = data.concat(Object.keys(received[1].data)); + } + data.forEach(d => { + expect(d).not.toMatch(/^context_\d/g); + }); + }); }); - let data = Object.keys(received[0].data); - if (received[1]) { - data = data.concat(Object.keys(received[1].data)); - } - data.forEach(d => { - expect(d).not.toMatch(/^context_\d/g); - }); - }); }); - }); }); describe("context ip", () => { - it("should set context_ip to context.ip if present", () => { - eventTypes.forEach(evType => { - let i = input(evType); - i.message.context.ip = "new_ip"; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toBe("string"); - expect( - received[0].data[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toEqual("new_ip"); - - if (received[1]) { - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toBe("string"); - expect( - received[1].data[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toEqual("new_ip"); - } - }); + it("should set context_ip to context.ip if present", () => { + eventTypes.forEach(evType => { + let i = input(evType); + i.message.context.ip = "new_ip"; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toBe("string"); + expect( + received[0].data[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toEqual("new_ip"); + + if (received[1]) { + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toBe("string"); + expect( + received[1].data[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toEqual("new_ip"); + } + }); + }); }); - }); - - it("should set context_ip to request_ip if context.ip not present", () => { - eventTypes.forEach(evType => { - let i = input(evType); - delete i.message.context.ip; - i.message.request_ip = "requested_ip"; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toBe("string"); - expect( - received[0].data[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toEqual("requested_ip"); - if (received[1]) { - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toBe("string"); - expect( - received[1].data[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toEqual("requested_ip"); - } - }); + + it("should set context_ip to request_ip if context.ip not present", () => { + eventTypes.forEach(evType => { + let i = input(evType); + delete i.message.context.ip; + i.message.request_ip = "requested_ip"; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toBe("string"); + expect( + received[0].data[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toEqual("requested_ip"); + if (received[1]) { + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toBe("string"); + expect( + received[1].data[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toEqual("requested_ip"); + } + }); + }); }); - }); }); describe("remove rudder property if rudder property is null", () => { - it("should remove context_ip set by user in properties if missing in event", () => { - eventTypes.forEach(evType => { - let i = input(evType); - delete i.message.context.ip; - delete i.message.request_ip; - const propsKey = propsKeyMap[evType]; - _.set(i.message, `${propsKey}.context_ip`, "test prop"); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "context_ip") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "context_ip") - ); - }); + it("should remove context_ip set by user in properties if missing in event", () => { + eventTypes.forEach(evType => { + let i = input(evType); + delete i.message.context.ip; + delete i.message.request_ip; + const propsKey = propsKeyMap[evType]; + _.set(i.message, `${propsKey}.context_ip`, "test prop"); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "context_ip") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "context_ip") + ); + }); + }); }); - }); }); + describe("remove any property if event is object ", () => { - it("should remove any property if event is object", () => { - eventTypes.forEach(evType => { - let i = input(evType); - i.message.channel = {}; - i.message.event = {}; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "channel") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "channel") - ); - }); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "event_text") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "event_text") - ); - }); - i.message.channel = { channel: "android" }; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "channel") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "channel") - ); - }); + it("should remove any property if event is object", () => { + eventTypes.forEach(evType => { + let i = input(evType); + i.message.channel = {}; + i.message.event = {}; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "channel") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "channel") + ); + }); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "event_text") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "event_text") + ); + }); + i.message.channel = {channel: "android"}; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "channel") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "channel") + ); + }); + }); }); - }); }); describe("store full rudder event", () => { - it("should store if configured in dest settings", () => { - eventTypes.forEach(evType => { - let i = input(evType); - _.set(i.destination, `Config.storeFullEvent`, true); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - const columnName = integrationCasedString( - integrations[index], - "rudder_event" - ); - - expect(received[0].metadata.columns).toHaveProperty(columnName); - expect(received[0].metadata.columns[columnName]).toEqual( - fullEventColumnTypeByProvider[integrations[index]] - ); - expect(received[0].data[columnName]).toEqual(JSON.stringify(i.message)); - - if (received[1]) { - expect(received[1].metadata.columns).not.toHaveProperty(columnName); - expect(received[1].data).not.toHaveProperty(columnName); - } - }); + it("should store if configured in dest settings", () => { + eventTypes.forEach(evType => { + let i = input(evType); + _.set(i.destination, `Config.storeFullEvent`, true); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + const columnName = integrationCasedString( + integrations[index], + "rudder_event" + ); + + expect(received[0].metadata.columns).toHaveProperty(columnName); + expect(received[0].metadata.columns[columnName]).toEqual( + fullEventColumnTypeByProvider[integrations[index]] + ); + expect(received[0].data[columnName]).toEqual(JSON.stringify(i.message)); + + if (received[1]) { + expect(received[1].metadata.columns).not.toHaveProperty(columnName); + expect(received[1].data).not.toHaveProperty(columnName); + } + }); + }); }); - }); }); describe("rudder reserved columns", () => { - it("should not accept rudder reserved column names from user in properties, traits etc", () => { - eventTypes.forEach(evType => { - let i = input(evType); - - const delProps = [ - "message.channel", - "message.timestamp", - "message.originalTimestamp" - ]; - const setProps = [ - "message.properties.channel", - "message.traits.channel", - "message.properties.timestamp", - "message.traits.timestamp", - "message.properties.originalTimestamp", - "message.traits.originalTimestamp" - ]; - - const checkProps = ["channel", "timestamp", "original_timestamp"]; - - delProps.forEach(prop => _.unset(i, prop)); - setProps.forEach(prop => _.set(i, prop, "random value")); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - checkProps.forEach(k => { - k = integrationCasedString(integrations[index], k); - expect(received[0].metadata.columns).not.toHaveProperty(k); - expect(received[0].data).not.toHaveProperty(k); - if (received[1]) { - expect(received[1].metadata.columns).not.toHaveProperty(k); - expect(received[1].data).not.toHaveProperty(k); - } + it("should not accept rudder reserved column names from user in properties, traits etc", () => { + eventTypes.forEach(evType => { + let i = input(evType); + + const delProps = [ + "message.channel", + "message.timestamp", + "message.originalTimestamp" + ]; + const setProps = [ + "message.properties.channel", + "message.traits.channel", + "message.properties.timestamp", + "message.traits.timestamp", + "message.properties.originalTimestamp", + "message.traits.originalTimestamp" + ]; + + const checkProps = ["channel", "timestamp", "original_timestamp"]; + + delProps.forEach(prop => _.unset(i, prop)); + setProps.forEach(prop => _.set(i, prop, "random value")); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + checkProps.forEach(k => { + k = integrationCasedString(integrations[index], k); + expect(received[0].metadata.columns).not.toHaveProperty(k); + expect(received[0].data).not.toHaveProperty(k); + if (received[1]) { + expect(received[1].metadata.columns).not.toHaveProperty(k); + expect(received[1].data).not.toHaveProperty(k); + } + }); + }); }); - }); }); - }); }); describe("id column datatype for users table", () => { - it("should set id column datatype for users as one received and not always string", () => { - let i = input("identify"); - i.message.userId = 100; //integer - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "user_id") - ] - ).toEqual("int"); - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "id") - ] - ).toEqual("int"); - }); + it("should set id column datatype for users as one received and not always string", () => { + let i = input("identify"); + i.message.userId = 100; //integer + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "user_id") + ] + ).toEqual("int"); + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "id") + ] + ).toEqual("int"); + }); - i = input("identify"); - i.message.userId = 1.1; //float - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "user_id") - ] - ).toEqual("float"); - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "id") - ] - ).toEqual("float"); + i = input("identify"); + i.message.userId = 1.1; //float + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "user_id") + ] + ).toEqual("float"); + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "id") + ] + ).toEqual("float"); + }); }); - }); }); describe("handle leading underscores in properties", () => { - it("should not remove leading underscores in properties", () => { - let i = input("track"); - i.message.properties = { - _timestamp: "1", - __timestamp: "2", - __timestamp_new: "3" - }; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "_timestamp") - ); - expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "__timestamp") - ); - expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "__timestamp_new") - ); - expect(received[1].data).toHaveProperty( - integrationCasedString(integrations[index], "_timestamp") - ); - expect(received[1].data).toHaveProperty( - integrationCasedString(integrations[index], "__timestamp") - ); - expect(received[1].data).toHaveProperty( - integrationCasedString(integrations[index], "__timestamp_new") - ); + it("should not remove leading underscores in properties", () => { + let i = input("track"); + i.message.properties = { + _timestamp: "1", + __timestamp: "2", + __timestamp_new: "3" + }; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[1].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "_timestamp") + ); + expect(received[1].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "__timestamp") + ); + expect(received[1].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "__timestamp_new") + ); + expect(received[1].data).toHaveProperty( + integrationCasedString(integrations[index], "_timestamp") + ); + expect(received[1].data).toHaveProperty( + integrationCasedString(integrations[index], "__timestamp") + ); + expect(received[1].data).toHaveProperty( + integrationCasedString(integrations[index], "__timestamp_new") + ); + }); }); - }); }); describe("handle recordId from cloud sources", () => { - it("should not set id based on recordId if sourceCategory is missing", () => { - let i = input("track"); - i.message.recordId = 42; - if (i.metadata) delete i.metadata.sourceCategory; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect( - received[0].data[integrationCasedString(integrations[index], "id")] - ).toEqual(i.message.messageId); - expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[1].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect( - received[1].data[integrationCasedString(integrations[index], "id")] - ).toEqual(i.message.messageId); + it("should not set id based on recordId if sourceCategory is missing", () => { + let i = input("track"); + i.message.recordId = 42; + if (i.metadata) delete i.metadata.sourceCategory; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect( + received[0].data[integrationCasedString(integrations[index], "id")] + ).toEqual(i.message.messageId); + expect(received[1].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[1].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect( + received[1].data[integrationCasedString(integrations[index], "id")] + ).toEqual(i.message.messageId); + }); }); - }); - - it("should not set id based on recordId if source version is missing", () => { - let i = input("track"); - i.message.recordId = 42; - i.metadata = { sourceCategory: "cloud" }; - if (i.message.context.sources) delete i.message.context.sources.version; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect( - received[0].data[integrationCasedString(integrations[index], "id")] - ).toEqual(i.message.messageId); - expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[1].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect( - received[1].data[integrationCasedString(integrations[index], "id")] - ).toEqual(i.message.messageId); + + it("should not set id based on recordId if source version is missing", () => { + let i = input("track"); + i.message.recordId = 42; + i.metadata = {sourceCategory: "cloud"}; + if (i.message.context.sources) delete i.message.context.sources.version; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect( + received[0].data[integrationCasedString(integrations[index], "id")] + ).toEqual(i.message.messageId); + expect(received[1].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[1].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect( + received[1].data[integrationCasedString(integrations[index], "id")] + ).toEqual(i.message.messageId); + }); }); - }); - - it("should not set id based on recordId if event type is not track", () => { - let i = input("page"); - i.message.recordId = 42; - i.metadata = { sourceCategory: "cloud" }; - if (i.message.context.sources) delete i.message.context.sources.version; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect( - received[0].data[integrationCasedString(integrations[index], "id")] - ).toEqual(i.message.messageId); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "id") - ] - ).toEqual("string"); + + it("should not set id based on recordId if event type is not track", () => { + let i = input("page"); + i.message.recordId = 42; + i.metadata = {sourceCategory: "cloud"}; + if (i.message.context.sources) delete i.message.context.sources.version; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect( + received[0].data[integrationCasedString(integrations[index], "id")] + ).toEqual(i.message.messageId); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "id") + ] + ).toEqual("string"); + }); }); - }); - - it("should set id based on recordId when sourceCategory is cloud and has source version", () => { - let i = input("track"); - i.message.recordId = 42; - i.message.properties.recordId = "anotherRecordId"; - i.metadata = { sourceCategory: "cloud" }; - i.message.context.sources = { version: 1.12 }; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "record_id") - ] - ).toEqual("string"); - expect( - received[0].data[ - integrationCasedString(integrations[index], "record_id") - ] - ).toBe("42"); - - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "id") - ] - ).toEqual("int"); - expect( - received[1].data[integrationCasedString(integrations[index], "id")] - ).toBe(42); - expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[1].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); + + it("should set id based on recordId when sourceCategory is cloud and has source version", () => { + let i = input("track"); + i.message.recordId = 42; + i.message.properties.recordId = "anotherRecordId"; + i.metadata = {sourceCategory: "cloud"}; + i.message.context.sources = {version: 1.12}; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "record_id") + ] + ).toEqual("string"); + expect( + received[0].data[ + integrationCasedString(integrations[index], "record_id") + ] + ).toBe("42"); + + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "id") + ] + ).toEqual("int"); + expect( + received[1].data[integrationCasedString(integrations[index], "id")] + ).toBe(42); + expect(received[1].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[1].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + }); }); - }); - - it("should set id based on recordId when sourceCategory is singer-protocol and has source version", () => { - let i = input("track"); - i.message.recordId = 42; - i.message.properties.recordId = "anotherRecordId"; - i.metadata = { sourceCategory: "singer-protocol" }; - i.message.context.sources = { version: 1.12 }; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "record_id") - ] - ).toEqual("string"); - expect( - received[0].data[ - integrationCasedString(integrations[index], "record_id") - ] - ).toBe("42"); - - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "id") - ] - ).toEqual("int"); - expect( - received[1].data[integrationCasedString(integrations[index], "id")] - ).toBe(42); - expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[1].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); + + it("should set id based on recordId when sourceCategory is singer-protocol and has source version", () => { + let i = input("track"); + i.message.recordId = 42; + i.message.properties.recordId = "anotherRecordId"; + i.metadata = {sourceCategory: "singer-protocol"}; + i.message.context.sources = {version: 1.12}; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "record_id") + ] + ).toEqual("string"); + expect( + received[0].data[ + integrationCasedString(integrations[index], "record_id") + ] + ).toBe("42"); + + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "id") + ] + ).toEqual("int"); + expect( + received[1].data[integrationCasedString(integrations[index], "id")] + ).toBe(42); + expect(received[1].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[1].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + }); }); - }); - - it("should throw error when sourceCategory is cloud and has source version and recordId is missing", () => { - let i = input("track"); - i.message.properties.recordId = "anotherRecordId"; - i.metadata = { sourceCategory: "cloud" }; - i.message.context.sources = { version: 1.12 }; - transformers.forEach((transformer, index) => { - expect(() => transformer.process(i)).toThrow( - "recordId cannot be empty for cloud sources events" - ); + + it("should throw error when sourceCategory is cloud and has source version and recordId is missing", () => { + let i = input("track"); + i.message.properties.recordId = "anotherRecordId"; + i.metadata = {sourceCategory: "cloud"}; + i.message.context.sources = {version: 1.12}; + transformers.forEach((transformer, index) => { + expect(() => transformer.process(i)).toThrow( + "recordId cannot be empty for cloud sources events" + ); + }); }); - }); }); describe("handle level three nested events from sources", () => { - it("should stringify event properties whose level ge 3 for cloud sources", () => { - const i = input("track"); - i.metadata = { sourceCategory: "cloud" }; - i.message.properties = Object.assign(i.message.properties, { - n0: { - prop0: "prop level 0", - n1: { - prop1: "prop level 1", - n2: { - prop2: "prop level 2", - n3: { - prop3: "prop level 3", - n4: { prop4: "prop level 4" } + it("should stringify event properties whose level ge 3 for cloud sources", () => { + const i = input("track"); + i.metadata = {sourceCategory: "cloud"}; + i.message.properties = Object.assign(i.message.properties, { + n0: { + prop0: "prop level 0", + n1: { + prop1: "prop level 1", + n2: { + prop2: "prop level 2", + n3: { + prop3: "prop level 3", + n4: {prop4: "prop level 4"} + } + } + } } - } - } - } - }); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") - ); - expect(received[1].data).not.toHaveProperty( - integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") - ); + }); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[1].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") + ); + expect(received[1].data).not.toHaveProperty( + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") + ); + }); }); - }); - - it("should not stringify event properties whose level ge 3 for non cloud sources", () => { - const i = input("track"); - i.message.properties = Object.assign(i.message.properties, { - n0: { - prop0: "prop level 0", - n1: { - prop1: "prop level 1", - n2: { - prop2: "prop level 2", - n3: { - prop3: "prop level 3", - n4: { prop4: "prop level 4" } + + it("should not stringify event properties whose level ge 3 for non cloud sources", () => { + const i = input("track"); + i.message.properties = Object.assign(i.message.properties, { + n0: { + prop0: "prop level 0", + n1: { + prop1: "prop level 1", + n2: { + prop2: "prop level 2", + n3: { + prop3: "prop level 3", + n4: {prop4: "prop level 4"} + } + } + } } - } - } - } - }); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") - ); - expect(received[1].data).toHaveProperty( - integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") - ); + }); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[1].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") + ); + expect(received[1].data).toHaveProperty( + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") + ); + }); }); - }); }); describe("Handle no of columns in an event", () => { - it("should throw an error if no of columns are more than 200", () => { - const i = input("track"); - transformers - .filter((transformer, index) => integrations[index] !== "s3_datalake" && integrations[index] !== "gcs_datalake" && integrations[index] !== "azure_datalake") - .forEach((transformer, index) => { - i.message.properties = largeNoOfColumnsevent; - expect(() => transformer.process(i)).toThrow( - "transformer: Too many columns outputted from the event" - ); - }); - }); - - it("should not throw an error if no of columns are more than 200 and the event is from rudder-sources", () => { - const i = input("track"); - transformers.forEach((transformer, index) => { - i.message.channel = "sources"; - i.message.properties = largeNoOfColumnsevent; - expect(() => transformer.process(i)).not.toThrow(); + it("should throw an error if no of columns are more than 200", () => { + const i = input("track"); + transformers + .filter((transformer, index) => integrations[index] !== "s3_datalake" && integrations[index] !== "gcs_datalake" && integrations[index] !== "azure_datalake") + .forEach((transformer, index) => { + i.message.properties = largeNoOfColumnsevent; + expect(() => transformer.process(i)).toThrow( + "transformer: Too many columns outputted from the event" + ); + }); + }); + + it("should not throw an error if no of columns are more than 200 and the event is from rudder-sources", () => { + const i = input("track"); + transformers.forEach((transformer, index) => { + i.message.channel = "sources"; + i.message.properties = largeNoOfColumnsevent; + expect(() => transformer.process(i)).not.toThrow(); + }); }); - }); }); describe("Add auto generated messageId for events missing it", () => { - it("should remove context_ip set by user in properties if missing in event", () => { - eventTypes.forEach(evType => { - let i = input(evType); - delete i.message.messageId; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "id") - ); - expect(received[0].data).toHaveProperty( - integrationCasedString(integrations[index], "id") - ); - expect( - received[0].data[integrationCasedString(integrations[index], "id")] - ).toMatch(/auto-.*/); - }); + it("should remove context_ip set by user in properties if missing in event", () => { + eventTypes.forEach(evType => { + let i = input(evType); + delete i.message.messageId; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "id") + ); + expect(received[0].data).toHaveProperty( + integrationCasedString(integrations[index], "id") + ); + expect( + received[0].data[integrationCasedString(integrations[index], "id")] + ).toMatch(/auto-.*/); + }); + }); }); - }); }); describe("Add receivedAt for events missing it", () => { - it("should remove context_ip set by user in properties if missing in event", () => { - eventTypes.forEach(evType => { - let i = input(evType); - delete i.message.receivedAt; - delete i.message.received_at; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "received_at") - ); - expect(received[0].data).toHaveProperty( - integrationCasedString(integrations[index], "received_at") - ); - }); + it("should remove context_ip set by user in properties if missing in event", () => { + eventTypes.forEach(evType => { + let i = input(evType); + delete i.message.receivedAt; + delete i.message.received_at; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "received_at") + ); + expect(received[0].data).toHaveProperty( + integrationCasedString(integrations[index], "received_at") + ); + }); + }); }); - }); }); describe("Integration options", () => { - describe("Destination config options", () => { - destConfig.scenarios().forEach(scenario => { - it(scenario.name, () => { - if (scenario.skipUsersTable !== null) { - scenario.event.destination.Config.skipUsersTable = scenario.skipUsersTable - } - if (scenario.skipTracksTable !== null) { - scenario.event.destination.Config.skipTracksTable = scenario.skipTracksTable - } + describe("track", () => { + it("should generate two events for every track call", () => { + const i = opInput("track"); + transformers.forEach((transformer, index) => { + const {jsonPaths} = i.destination.Config; + if (integrations[index] === "postgres") { + delete i.destination.Config.jsonPaths; + } + const received = transformer.process(i); + i.destination.Config.jsonPaths = jsonPaths; + expect(received).toEqual(opOutput("track", integrations[index])); + }); + }); + }); - transformers.forEach((transformer, index) => { - const received = transformer.process(scenario.event); - expect(received).toHaveLength(scenario.expected.length); - for (const i in received) { - const evt = received[i]; - expect(evt.data.id ? evt.data.id : evt.data.ID).toEqual(scenario.expected[i].id); - expect(evt.metadata.table.toLowerCase()).toEqual(scenario.expected[i].table); - } + describe("users", () => { + it("should skip users when skipUsersTable is set", () => { + const i = opInput("users"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toEqual(opOutput("users", integrations[index])); + }); }); - }); }); - }); - describe("track", () => { - it("should generate two events for every track call", () => { - const i = opInput("track"); - transformers.forEach((transformer, index) => { - const {jsonPaths} = i.destination.Config; - if (integrations[index] === "postgres") { - delete i.destination.Config.jsonPaths; + + describe("json paths", () => { + const output = (config, provider) => { + switch (provider) { + case "rs": + return _.cloneDeep(config.output.rs); + case "bq": + return _.cloneDeep(config.output.bq); + case "postgres": + return _.cloneDeep(config.output.postgres); + case "snowflake": + return _.cloneDeep(config.output.snowflake); + default: + return _.cloneDeep(config.output.default); + } + } + + const testCases = [ + { + eventType: "aliases", + }, + { + eventType: "groups", + }, + { + eventType: "identifies", + }, + { + eventType: "pages", + }, + { + eventType: "screens", + }, + { + eventType: "tracks", + }, + { + eventType: "extract", + }, + ]; + + for (const testCase of testCases) { + transformers.forEach((transformer, index) => { + it(`new ${testCase.eventType} for ${integrations[index]}`, () => { + const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); + const input = _.cloneDeep(config.input); + const received = transformer.process(input); + expect(received).toEqual(output(config, integrations[index])); + }) + + it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { + const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); + const input = _.cloneDeep(config.input); + const received = transformer.process(input); + expect(received).toEqual(output(config, integrations[index])); + }) + }); } - const received = transformer.process(i); - i.destination.Config.jsonPaths = jsonPaths; - expect(received).toEqual(opOutput("track", integrations[index])); - }); }); - }); - - describe("users", () => { - it("should skip users when skipUsersTable is set", () => { - const i = opInput("users"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toEqual(opOutput("users", integrations[index])); - }); +}); + +describe("Destination config", () => { + describe("skipUsersTable, skipTracksTable", () => { + destConfig.scenarios().forEach(scenario => { + it(scenario.name, () => { + if (scenario.skipUsersTable !== null) { + scenario.event.destination.Config.skipUsersTable = scenario.skipUsersTable + } + if (scenario.skipTracksTable !== null) { + scenario.event.destination.Config.skipTracksTable = scenario.skipTracksTable + } + + transformers.forEach((transformer, index) => { + const received = transformer.process(scenario.event); + expect(received).toHaveLength(scenario.expected.length); + for (const i in received) { + const evt = received[i]; + expect(evt.data.id ? evt.data.id : evt.data.ID).toEqual(scenario.expected[i].id); + expect(evt.metadata.table.toLowerCase()).toEqual(scenario.expected[i].table); + } + }); + }); + }); }); - }); - - describe("json paths", () => { - const output = (config, provider) => { - switch (provider) { - case "rs": - return _.cloneDeep(config.output.rs); - case "bq": - return _.cloneDeep(config.output.bq); - case "postgres": - return _.cloneDeep(config.output.postgres); - case "snowflake": - return _.cloneDeep(config.output.snowflake); - default: - return _.cloneDeep(config.output.default); - } - } + describe('allowUsersContextTraits, allowEventContextTraits, underscoreDivideNumbers', () => { + describe("old destinations", () => { + it('without allowEventContextTraits', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: { + allowEventContextTraits: true + }, + }, + message: { + context: { + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + } + }, + event: "button clicked", + type: "track", + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + const events = [output[0], output[1]]; + const traitsToCheck = { + 'city': 'Disney', + 'country': 'USA', + 'email': 'mickey@disney.com', + 'firstname': 'Mickey' + }; + events.forEach(event => { + Object.entries(traitsToCheck).forEach(([trait, value]) => { + expect(output[0].data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + expect(output[1].data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + }); + }); + }); + }); + + it('with allowUsersContextTraits', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: { + allowUsersContextTraits: true + } + }, + message: { + context: { + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + }, + traits: { + lastname: "Mouse" + }, + type: "identify", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + const events = [output[0], output[1]]; // identifies and users event + const traitsToCheck = { + 'city': 'Disney', + 'country': 'USA', + 'email': 'mickey@disney.com', + 'firstname': 'Mickey' + }; + events.forEach(event => { + Object.entries(traitsToCheck).forEach(([trait, value]) => { + expect(event.data[integrationCasedString(integrations[index], trait)]).toEqual(value); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + }); + }); + }); + }); + + it('without underscoreDivideNumbers', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: { + underscoreDivideNumbers: true + }, + }, + message: { + context: { + 'attribute v3': 'some-value' + }, + event: "button clicked v2", + type: "track", + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v_2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); + }); + }); + }); + describe("new destinations", () => { + it('without allowEventContextTraits', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: {}, + }, + message: { + context: { + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + } + }, + event: "button clicked", + type: "track", + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + const traitsToCheck = { + 'city': 'Disney', + 'country': 'USA', + 'email': 'mickey@disney.com', + 'firstname': 'Mickey' + }; + Object.entries(traitsToCheck).forEach(([trait, value]) => { + expect(output[0].data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + expect(output[1].data).not.toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + expect(output[1].metadata.columns).not.toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + }); + }); + }); + + it('without allowUsersContextTraits', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: {} + }, + message: { + context: { + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + }, + traits: { + lastname: "Mouse" + }, + type: "identify", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const received = transformer.process(event); + const events = [received[0], received[1]]; // identifies and users event + const traitsToCheck = { + 'city': 'Disney', + 'country': 'USA', + 'email': 'mickey@disney.com', + 'firstname': 'Mickey' + }; + events.forEach(event => { + Object.entries(traitsToCheck).forEach(([trait, value]) => { + expect(event.data).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); + }); + }); + }); + }); + + it('without underscoreDivideNumbers', () => { + transformers.forEach((transformer, index) => { + const event = { + destination: { + Config: {}, + }, + message: { + context: { + 'attribute v3': 'some-value' + }, + event: "button clicked v2", + type: "track", + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + } + const output = transformer.process(event); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); + }); + }); + }); + }); +}); + +describe("validTimestamp", () => { const testCases = [ - { - eventType: "aliases", - }, - { - eventType: "groups", - }, - { - eventType: "identifies", - }, - { - eventType: "pages", - }, - { - eventType: "screens", - }, - { - eventType: "tracks", - }, - { - eventType: "extract", - }, + { + name: "undefined input should return false", + input: undefined, + expected: false, + }, + { + name: "negative year and time input should return false #1", + input: '-0001-11-30T00:00:00+0000', + expected: false, + }, + { + name: "negative year and time input should return false #2", + input: '-2023-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "negative year and time input should return false #3", + input: '-1900-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "positive year and time input should return false", + input: '+2023-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "valid timestamp input should return true", + input: '2023-06-14T05:23:59.244Z', + expected: true, + }, + { + name: "non-date string input should return false", + input: 'abc', + expected: false, + }, + { + name: "malicious string input should return false", + input: '%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216Windows%u2216win%u002ein', + expected: false, + }, + { + name: "empty string input should return false", + input: '', + expected: false, + }, + { + name: "valid date input should return true", + input: '2023-06-14', + expected: true, + }, + { + name: "time-only input should return false", + input: '05:23:59.244Z', + expected: false, + }, + { + name: "non-string input should return false", + input: {abc: 123}, + expected: false, + }, + { + name: "object with toString method input should return false", + input: { + toString: '2023-06-14T05:23:59.244Z' + }, + expected: false, + }, ]; - for (const testCase of testCases) { - transformers.forEach((transformer, index) => { - it(`new ${testCase.eventType} for ${integrations[index]}`, () => { - const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); - const input = _.cloneDeep(config.input); - const received = transformer.process(input); - expect(received).toEqual(output(config, integrations[index])); - }) - - it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { - const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); - const input = _.cloneDeep(config.input); - const received = transformer.process(input); - expect(received).toEqual(output(config, integrations[index])); - }) - }); + it(`should return ${testCase.expected} for ${testCase.name}`, () => { + expect(validTimestamp(testCase.input)).toEqual(testCase.expected); + }); } - }); }); -describe("validTimestamp", () => { - const testCases = [ - { - name: "undefined input should return false", - input: undefined, - expected: false, - }, - { - name: "negative year and time input should return false #1", - input: '-0001-11-30T00:00:00+0000', - expected: false, - }, - { - name: "negative year and time input should return false #2", - input: '-2023-06-14T05:23:59.244Z', - expected: false, - }, - { - name: "negative year and time input should return false #3", - input: '-1900-06-14T05:23:59.244Z', - expected: false, - }, - { - name: "positive year and time input should return false", - input: '+2023-06-14T05:23:59.244Z', - expected: false, - }, - { - name: "valid timestamp input should return true", - input: '2023-06-14T05:23:59.244Z', - expected: true, - }, - { - name: "non-date string input should return false", - input: 'abc', - expected: false, - }, - { - name: "malicious string input should return false", - input: '%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216Windows%u2216win%u002ein', - expected: false, - }, - { - name: "empty string input should return false", - input: '', - expected: false, - }, - { - name: "valid date input should return true", - input: '2023-06-14', - expected: true, - }, - { - name: "time-only input should return false", - input: '05:23:59.244Z', - expected: false, - }, - { - name: "non-string input should return false", - input: { abc: 123 }, - expected: false, - }, - { - name: "object with toString method input should return false", - input: { - toString: '2023-06-14T05:23:59.244Z' - }, - expected: false, - }, - ]; - - for (const testCase of testCases) { - it(`should return ${testCase.expected} for ${testCase.name}`, () => { - expect(validTimestamp(testCase.input)).toEqual(testCase.expected); - }); - } +describe("isBlank", () => { + const testCases = [ + { + name: "null", + input: null, + expected: true + }, + { + name: "empty string", + input: "", + expected: true + }, + { + name: "non-empty string", + input: "test", + expected: false + }, + { + name: "numeric value", + input: 1634762544, + expected: false + }, + { + name: "object with toString property", + input: { + toString: '2023-06-14T05:23:59.244Z' + }, + expected: false + }, + ]; + for (const testCase of testCases) { + it(`should return ${testCase.expected} for ${testCase.name}`, () => { + expect(isBlank(testCase.input)).toEqual(testCase.expected); + }); + } }); +describe("transformColumnName", () => { + describe('with Blendo Casing', () => { + const testCases = [ + { + description: 'should convert special characters other than "\\" or "$" to underscores', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'column@Name#1', + expected: 'column_name_1', + }, + { + description: 'should add underscore if name does not start with an alphabet or underscore', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: '1CComega', + expected: '_1ccomega', + }, + { + description: 'should handle non-ASCII characters by converting to underscores', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'Cízǔ', + expected: 'c_z_', + }, + { + description: 'should preserve "\\" and "$" characters', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'path_to_$1,00,000', + expected: 'path_to_$1_00_000', + }, + { + description: 'should transform CamelCase123Key to camelcase123key', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'CamelCase123Key', + expected: 'camelcase123key', + }, + { + description: 'should handle a mix of characters, numbers, and special characters', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'CamelCase123Key_with$special\\chars', + expected: 'camelcase123key_with$special\\chars', + }, + { + description: 'should handle names starting with a number', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: '1Column', + expected: '_1column', + }, + { + description: 'should limit length to 63 characters for postgres provider', + options: {integrationOptions: {useBlendoCasing: true}, provider: 'postgres'}, + input: 'a'.repeat(70), + expected: 'a'.repeat(63), + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformColumnName(options, input); + expect(result).toBe(expected); + }); + }); + }); + describe('without Blendo Casing', () => { + const testCases = [ + { + description: 'should remove symbols and join continuous letters and numbers with a single underscore', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '&4yasdfa(84224_fs9##_____*3q', + expected: '_4_yasdfa_84224_fs_9_3_q', + }, + { + description: 'should transform "omega" to "omega"', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega', + expected: 'omega', + }, + { + description: 'should transform "omega v2" to "omega_v_2"', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega v2', + expected: 'omega_v_2', + }, + { + description: 'should prepend underscore if name starts with a number', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '9mega', + expected: '_9_mega', + }, + { + description: 'should remove trailing special characters', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'mega&', + expected: 'mega', + }, + { + description: 'should replace special character in the middle with underscore', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'ome$ga', + expected: 'ome_ga', + }, + { + description: 'should not remove trailing $ character', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'omega$', + expected: 'omega', + }, + { + description: 'should handle spaces and special characters by converting to underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'ome_ ga', + expected: 'ome_ga', + }, + { + description: 'should handle multiple underscores and hyphens by reducing to single underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '9mega________-________90', + expected: '_9_mega_90', + }, + { + description: 'should handle non-ASCII characters by converting them to underscores', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'Cízǔ', + expected: 'c_z', + }, + { + description: 'should transform CamelCase123Key to camel_case_123_key', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'CamelCase123Key', + expected: 'camel_case_123_key', + }, + { + description: 'should handle numbers and commas in the input', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'path to $1,00,000', + expected: 'path_to_1_00_000', + }, + { + description: 'should return an empty string if input contains no valid characters', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: '@#$%', + expected: '', + }, + { + description: 'should remove underscores between letters and numbers when underscoreDivideNumbers is false', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'test123', + expected: 'test123', + }, + { + description: 'should keep underscores between letters and numbers when underscoreDivideNumbers is true', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'test123', + expected: 'test_123', + }, + { + description: 'should correctly handle multiple underscore-number sequences when underscoreDivideNumbers is false', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'abc123def456', + expected: 'abc123_def456', + }, + { + description: 'should avoid adding extra underscores when underscoreDivideNumbers is false', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: false + }, + input: 'abc_123_def_456', + expected: 'abc123_def456', + }, + { + description: 'should keep multiple underscore-number sequences when underscoreDivideNumbers is true', + options: { + integrationOptions: {useBlendoCasing: false}, + provider: 'postgres', + underscoreDivideNumbers: true + }, + input: 'abc123def456', + expected: 'abc_123_def_456', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformColumnName(options, input); + expect(result).toBe(expected); + }); + }); + }); +}) + +describe("transformTableName", () => { + describe('with Blendo Casing', () => { + const testCases = [ + { + description: 'should convert name to Blendo casing (lowercase) when Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'TableName123', + expected: 'tablename123', + }, + { + description: 'should trim spaces and convert to Blendo casing (lowercase) when Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: ' TableName ', + expected: 'tablename', + }, + { + description: 'should return an empty string when input is empty and Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: '', + expected: '', + }, + { + description: 'should handle names with special characters and convert to Blendo casing (lowercase)', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'Table@Name!', + expected: 'table@name!', + }, + { + description: 'should convert a mixed-case name to Blendo casing (lowercase) when Blendo casing is enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'CaMeLcAsE', + expected: 'camelcase', + }, + { + description: 'should keep an already lowercase name unchanged with Blendo casing enabled', + options: {integrationOptions: {useBlendoCasing: true}}, + input: 'lowercase', + expected: 'lowercase', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformTableName(options, input); + expect(result).toBe(expected); + }); + }); + }); -describe("isBlank", () => { - const testCases = [ - { - name: "null", - input: null, - expected: true - }, - { - name: "empty string", - input: "", - expected: true - }, - { - name: "non-empty string", - input: "test", - expected: false - }, - { - name: "numeric value", - input: 1634762544, - expected: false - }, - { - name: "object with toString property", - input: { - toString: '2023-06-14T05:23:59.244Z' - }, - expected: false - }, - ]; - - for (const testCase of testCases) { - it(`should return ${testCase.expected} for ${testCase.name}`, () => { - expect(isBlank(testCase.input)).toEqual(testCase.expected); + describe('without Blendo Casing', () => { + const testCases = [ + { + description: 'should remove symbols and join continuous letters and numbers with a single underscore', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '&4yasdfa(84224_fs9##_____*3q', + expected: '_4_yasdfa_84224_fs_9_3_q', + }, + { + description: 'should transform "omega" to "omega"', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega', + expected: 'omega', + }, + { + description: 'should transform "omega v2" to "omega_v_2"', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega v2', + expected: 'omega_v_2', + }, + { + description: 'should prepend underscore if name starts with a number', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '9mega', + expected: '_9_mega', + }, + { + description: 'should remove trailing special characters', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'mega&', + expected: 'mega', + }, + { + description: 'should replace special character in the middle with underscore', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'ome$ga', + expected: 'ome_ga', + }, + { + description: 'should not remove trailing $ character', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'omega$', + expected: 'omega', + }, + { + description: 'should handle spaces and special characters by converting to underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'ome_ ga', + expected: 'ome_ga', + }, + { + description: 'should handle multiple underscores and hyphens by reducing to single underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '9mega________-________90', + expected: '_9_mega_90', + }, + { + description: 'should handle non-ASCII characters by converting them to underscores', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'Cízǔ', + expected: 'c_z', + }, + { + description: 'should transform CamelCase123Key to camel_case_123_key', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'CamelCase123Key', + expected: 'camel_case_123_key', + }, + { + description: 'should handle numbers and commas in the input', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'path to $1,00,000', + expected: 'path_to_1_00_000', + }, + { + description: 'should return an empty string if input contains no valid characters', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: '@#$%', + expected: '', + }, + { + description: 'should remove underscores between letters and numbers when underscoreDivideNumbers is false', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'test123', + expected: 'test123', + }, + { + description: 'should keep underscores between letters and numbers when underscoreDivideNumbers is true', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'test123', + expected: 'test_123', + }, + { + description: 'should correctly handle multiple underscore-number sequences when underscoreDivideNumbers is false', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'abc123def456', + expected: 'abc123_def456', + }, + { + description: 'should avoid adding extra underscores when underscoreDivideNumbers is false', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: false}, + input: 'abc_123_def_456', + expected: 'abc123_def456', + }, + { + description: 'should keep multiple underscore-number sequences when underscoreDivideNumbers is true', + options: {integrationOptions: {useBlendoCasing: false}, underscoreDivideNumbers: true}, + input: 'abc123def456', + expected: 'abc_123_def_456', + }, + ]; + testCases.forEach(({description, options, input, expected}) => { + it(description, () => { + const result = transformTableName(options, input); + expect(result).toBe(expected); + }); + }); }); - } }); \ No newline at end of file diff --git a/test/integrations/destinations/gcs_datalake/processor/data.ts b/test/integrations/destinations/gcs_datalake/processor/data.ts index 46c7788709..dca1ad749a 100644 --- a/test/integrations/destinations/gcs_datalake/processor/data.ts +++ b/test/integrations/destinations/gcs_datalake/processor/data.ts @@ -62,6 +62,9 @@ export const data = [ syncFrequency: '30', tableSuffix: '', timeWindowLayout: '2006/01/02/15', + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true, }, Enabled: true, },