From d0c17e1b95419a074cf46b613497d5c8d76a3f7d Mon Sep 17 00:00:00 2001 From: achettyiitr Date: Thu, 19 Sep 2024 11:38:04 +0530 Subject: [PATCH] feat: bq custom partitioning --- src/warehouse/index.js | 14 +- .../warehouse/integration_options_events.js | 278 ++++++++++++++++++ .../jsonpaths/legacy/identifies.js | 174 +++++++++++ .../integrations/jsonpaths/new/identifies.js | 174 +++++++++++ test/__tests__/warehouse.test.js | 14 +- 5 files changed, 650 insertions(+), 4 deletions(-) diff --git a/src/warehouse/index.js b/src/warehouse/index.js index 3c2b04079d..71e574f52c 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -987,11 +987,23 @@ function processWarehouseMessage(message, options) { const usersEvent = { ...commonProps }; const usersColumnTypes = {}; + let userColumnMappingRules = whUserColumnMappingRules; + if (!isDataLakeProvider(options.provider)) { + userColumnMappingRules = { + ...userColumnMappingRules, + ...{ + sent_at: 'sentAt', + timestamp: 'timestamp', + original_timestamp: 'originalTimestamp', + }, + }; + } + setDataFromColumnMappingAndComputeColumnTypes( utils, usersEvent, message, - whUserColumnMappingRules, + userColumnMappingRules, usersColumnTypes, options, ); diff --git a/test/__tests__/data/warehouse/integration_options_events.js b/test/__tests__/data/warehouse/integration_options_events.js index 05a2d51abd..b32fb68705 100644 --- a/test/__tests__/data/warehouse/integration_options_events.js +++ b/test/__tests__/data/warehouse/integration_options_events.js @@ -880,6 +880,9 @@ const sampleEvents = { "email": "user123@email.com", "id": "user123", "phone": "+917836362334", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "received_at": "2020-01-24T06:29:02.403Z" }, "metadata": { @@ -906,6 +909,9 @@ const sampleEvents = { "id": "string", "phone": "string", "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -1074,6 +1080,9 @@ const sampleEvents = { "email": "user123@email.com", "id": "user123", "phone": "+917836362334", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "received_at": "2020-01-24T06:29:02.403Z" }, "metadata": { @@ -1101,6 +1110,9 @@ const sampleEvents = { "loaded_at": "datetime", "phone": "string", "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -1199,6 +1211,9 @@ const sampleEvents = { "EMAIL": "user123@email.com", "ID": "user123", "PHONE": "+917836362334", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", "RECEIVED_AT": "2020-01-24T06:29:02.403Z" }, "metadata": { @@ -1225,6 +1240,9 @@ const sampleEvents = { "ID": "string", "PHONE": "string", "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "ORIGINAL_TIMESTAMP": "datetime", "UUID_TS": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -1355,6 +1373,254 @@ const sampleEvents = { "table": "users" } } + ], + gcs_datalake: [ + { + "data": { + "timestamp": "2020-01-24T06:29:02.403Z", + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "user_id": "user123" + }, + "metadata": { + "columns": { + "timestamp": "datetime", + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + azure_datalake: [ + { + "data": { + "timestamp": "2020-01-24T06:29:02.403Z", + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "user_id": "user123" + }, + "metadata": { + "columns": { + "timestamp": "datetime", + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } ] } } @@ -1374,6 +1640,18 @@ function opOutput(eventType, provider) { return _.cloneDeep(sampleEvents[eventType].output.rs); case "bq": return _.cloneDeep(sampleEvents[eventType].output.bq); + case "gcs_datalake": + if (eventType === 'users') { + return _.cloneDeep(sampleEvents[eventType].output.gcs_datalake); + } else { + return _.cloneDeep(sampleEvents[eventType].output.default); + } + case "azure_datalake": + if (eventType === 'users') { + return _.cloneDeep(sampleEvents[eventType].output.azure_datalake); + } else { + return _.cloneDeep(sampleEvents[eventType].output.default); + } default: return _.cloneDeep(sampleEvents[eventType].output.default); } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js index 47b7f1209a..6164086eb8 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js @@ -230,6 +230,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -261,6 +264,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -374,6 +380,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -405,6 +414,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -519,6 +531,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -551,6 +566,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -664,6 +682,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -695,6 +716,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -808,6 +832,9 @@ module.exports = { "PHONE": "+917836362334", "RECEIVED_AT": "2020-01-24T06:29:02.403Z", "T_MAP_NESTED_MAP_N_1": "nested prop 1", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", "UP_MAP_NESTED_MAP_N_1": "nested prop 1" }, "metadata": { @@ -839,6 +866,9 @@ module.exports = { "RECEIVED_AT": "datetime", "T_MAP_NESTED_MAP_N_1": "string", "UP_MAP_NESTED_MAP_N_1": "string", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "ORIGINAL_TIMESTAMP": "datetime", "UUID_TS": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -846,5 +876,149 @@ module.exports = { } } ], + datalake: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js index 3d6164b430..f86e826a52 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js @@ -230,6 +230,9 @@ module.exports = { "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", "t_map_nested_map_n_1": "nested prop 1", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "up_map_nested_map_n_1": "nested prop 1" }, "metadata": { @@ -261,6 +264,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map_n_1": "string", "up_map_nested_map_n_1": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -373,6 +379,9 @@ module.exports = { "id": "user123", "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" }, @@ -405,6 +414,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map": "json", "up_map_nested_map": "json", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -518,6 +530,9 @@ module.exports = { "id": "user123", "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" }, @@ -551,6 +566,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map": "string", "up_map_nested_map": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -663,6 +681,9 @@ module.exports = { "id": "user123", "phone": "+917836362334", "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" }, @@ -695,6 +716,9 @@ module.exports = { "received_at": "datetime", "t_map_nested_map": "json", "up_map_nested_map": "json", + "sent_at": "datetime", + "timestamp": "datetime", + "original_timestamp": "datetime", "uuid_ts": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -807,6 +831,9 @@ module.exports = { "ID": "user123", "PHONE": "+917836362334", "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", "T_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}" }, @@ -839,6 +866,9 @@ module.exports = { "RECEIVED_AT": "datetime", "T_MAP_NESTED_MAP": "json", "UP_MAP_NESTED_MAP": "json", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "ORIGINAL_TIMESTAMP": "datetime", "UUID_TS": "datetime" }, "receivedAt": "2020-01-24T11:59:02.403+05:30", @@ -846,5 +876,149 @@ module.exports = { } } ], + datalake: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], } } diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 83b24aee15..1ee887633f 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -1058,7 +1058,7 @@ describe("Integration options", () => { }); describe("json paths", () => { - const output = (config, provider) => { + const output = (eventType, config, provider) => { switch (provider) { case "rs": return _.cloneDeep(config.output.rs); @@ -1068,6 +1068,14 @@ describe("Integration options", () => { return _.cloneDeep(config.output.postgres); case "snowflake": return _.cloneDeep(config.output.snowflake); + case "s3_datalake": + case "gcs_datalake": + case "azure_datalake": + if (eventType === 'identifies') { + return _.cloneDeep(config.output.datalake); + } else { + return _.cloneDeep(config.output.default); + } default: return _.cloneDeep(config.output.default); } @@ -1103,14 +1111,14 @@ describe("Integration options", () => { const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(config, integrations[index])); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(config, integrations[index])); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) }); }