diff --git a/src/warehouse/index.js b/src/warehouse/index.js index 54368da9fc..4afa8f72c2 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -1001,11 +1001,23 @@ function processWarehouseMessage(message, options) { const usersEvent = { ...commonProps }; const usersColumnTypes = {}; + let userColumnMappingRules = whUserColumnMappingRules; + if (!isDataLakeProvider(options.provider)) { + userColumnMappingRules = { + ...userColumnMappingRules, + ...{ + sent_at: 'sentAt', + timestamp: 'timestamp', + original_timestamp: 'originalTimestamp', + }, + }; + } + setDataFromColumnMappingAndComputeColumnTypes( utils, usersEvent, message, - whUserColumnMappingRules, + userColumnMappingRules, usersColumnTypes, options, ); diff --git a/test/__tests__/data/warehouse/integration_options_events.js b/test/__tests__/data/warehouse/integration_options_events.js index bb068d4857..35eafb6a9b 100644 --- a/test/__tests__/data/warehouse/integration_options_events.js +++ b/test/__tests__/data/warehouse/integration_options_events.js @@ -885,6 +885,9 @@ const sampleEvents = { "email": "user123@email.com", "id": "user123", "phone": "+917836362334", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "received_at": "2020-01-24T06:29:02.403Z" }, "metadata": { @@ -911,6 +914,9 @@ const sampleEvents = { "id": "string", "phone": "string", "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -1079,6 +1085,9 @@ const sampleEvents = { "email": "user123@email.com", "id": "user123", "phone": "+917836362334", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "received_at": "2020-01-24T06:29:02.403Z" }, "metadata": { @@ -1106,6 +1115,9 @@ const sampleEvents = { "loaded_at": "datetime", "phone": "string", "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -1204,6 +1216,9 @@ const sampleEvents = { "EMAIL": "user123@email.com", "ID": "user123", "PHONE": "+917836362334", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", "RECEIVED_AT": "2020-01-24T06:29:02.403Z" }, "metadata": { @@ -1230,6 +1245,9 @@ const sampleEvents = { "ID": "string", "PHONE": "string", "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "ORIGINAL_TIMESTAMP": "datetime", "UUID_TS": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -1360,6 +1378,254 @@ const sampleEvents = { "table": "users" } } + ], + gcs_datalake: [ + { + "data": { + "timestamp": "2020-01-24T06:29:02.403Z", + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "user_id": "user123" + }, + "metadata": { + "columns": { + "timestamp": "datetime", + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + azure_datalake: [ + { + "data": { + "timestamp": "2020-01-24T06:29:02.403Z", + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "user_id": "user123" + }, + "metadata": { + "columns": { + "timestamp": "datetime", + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } ] } } @@ -1379,6 +1645,18 @@ function opOutput(eventType, provider) { return _.cloneDeep(sampleEvents[eventType].output.rs); case "bq": return _.cloneDeep(sampleEvents[eventType].output.bq); + case "gcs_datalake": + if (eventType === 'users') { + return _.cloneDeep(sampleEvents[eventType].output.gcs_datalake); + } else { + return _.cloneDeep(sampleEvents[eventType].output.default); + } + case "azure_datalake": + if (eventType === 'users') { + return _.cloneDeep(sampleEvents[eventType].output.azure_datalake); + } else { + return _.cloneDeep(sampleEvents[eventType].output.default); + } default: return _.cloneDeep(sampleEvents[eventType].output.default); } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js index 14b01d4cda..42f5cccf49 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js @@ -233,6 +233,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -264,6 +267,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -377,6 +383,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -408,6 +417,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -522,6 +534,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -554,6 +569,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -667,6 +685,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -698,6 +719,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -811,6 +835,9 @@ module.exports = { "PHONE": "+917836362334", "RECEIVED_AT": "2020-01-24T06:29:02.403Z", "T_MAP_NESTED_MAP_N_1": "nested prop 1", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", "UP_MAP_NESTED_MAP_N_1": "nested prop 1" }, "metadata": { @@ -842,6 +869,9 @@ module.exports = { "RECEIVED_AT": "datetime", "T_MAP_NESTED_MAP_N_1": "string", "UP_MAP_NESTED_MAP_N_1": "string", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "ORIGINAL_TIMESTAMP": "datetime", "UUID_TS": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -849,5 +879,149 @@ module.exports = { } } ], + datalake: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js index 884770864a..89fcc23cd5 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js @@ -233,6 +233,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -264,6 +267,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -376,6 +382,9 @@ module.exports = { "id": "user123", "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" }, @@ -408,6 +417,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map": "json", "up_map_nested_map": "json", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -521,6 +533,9 @@ module.exports = { "id": "user123", "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" }, @@ -554,6 +569,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map": "string", "up_map_nested_map": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -666,6 +684,9 @@ module.exports = { "id": "user123", "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" }, @@ -698,6 +719,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map": "json", "up_map_nested_map": "json", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -810,6 +834,9 @@ module.exports = { "ID": "user123", "PHONE": "+917836362334", "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", "T_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}" }, @@ -842,6 +869,9 @@ module.exports = { "RECEIVED_AT": "datetime", "T_MAP_NESTED_MAP": "json", "UP_MAP_NESTED_MAP": "json", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "ORIGINAL_TIMESTAMP": "datetime", "UUID_TS": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -849,5 +879,149 @@ module.exports = { } } ], + datalake: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], } } diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 9099566c60..6bde2e9eb2 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -1035,7 +1035,7 @@ describe("Integration options", () => { }); describe("json paths", () => { - const output = (config, provider) => { + const output = (eventType, config, provider) => { switch (provider) { case "rs": return _.cloneDeep(config.output.rs); @@ -1045,6 +1045,14 @@ describe("Integration options", () => { return _.cloneDeep(config.output.postgres); case "snowflake": return _.cloneDeep(config.output.snowflake); + case "s3_datalake": + case "gcs_datalake": + case "azure_datalake": + if (eventType === 'identifies') { + return _.cloneDeep(config.output.datalake); + } else { + return _.cloneDeep(config.output.default); + } default: return _.cloneDeep(config.output.default); } @@ -1080,14 +1088,14 @@ describe("Integration options", () => { const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(config, integrations[index])); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(config, integrations[index])); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) }); }