Skip to content

Commit

Permalink
fix: populate users fields for sentAt, timestamp and originalTimestamp (
Browse files Browse the repository at this point in the history 
#3753)
  • Loading branch information
@achettyiitr
achettyiitr authored Sep 24, 2024
1 parent 386d2ab commit f50effe
Show file tree
Hide file tree
Showing 5 changed files with 650 additions and 4 deletions.
14 changes: 13 additions & 1 deletion src/warehouse/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1001,11 +1001,23 @@ function processWarehouseMessage(message, options) {

const usersEvent = { ...commonProps };
const usersColumnTypes = {};
let userColumnMappingRules = whUserColumnMappingRules;
if (!isDataLakeProvider(options.provider)) {
userColumnMappingRules = {
...userColumnMappingRules,
...{
sent_at: 'sentAt',
timestamp: 'timestamp',
original_timestamp: 'originalTimestamp',
},
};
}

setDataFromColumnMappingAndComputeColumnTypes(
utils,
usersEvent,
message,
whUserColumnMappingRules,
userColumnMappingRules,
usersColumnTypes,
options,
);
Expand Down
278 changes: 278 additions & 0 deletions test/__tests__/data/warehouse/integration_options_events.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -885,6 +885,9 @@ const sampleEvents = {
"email": "[email protected]",
"id": "user123",
"phone": "+917836362334",
"sent_at": "2021-01-03T17:02:53.195Z",
"original_timestamp": "2020-01-24T06:29:02.364Z",
"timestamp": "2020-01-24T06:29:02.403Z",
"received_at": "2020-01-24T06:29:02.403Z"
},
"metadata": {
Expand All @@ -911,6 +914,9 @@ const sampleEvents = {
"id": "string",
"phone": "string",
"received_at": "datetime",
"sent_at": "datetime",
"timestamp": "datetime",
"original_timestamp": "datetime",
"uuid_ts": "datetime"
},
"receivedAt": "2020-01-24T11:59:02.403+05:30",
Expand Down Expand Up @@ -1079,6 +1085,9 @@ const sampleEvents = {
"email": "[email protected]",
"id": "user123",
"phone": "+917836362334",
"sent_at": "2021-01-03T17:02:53.195Z",
"original_timestamp": "2020-01-24T06:29:02.364Z",
"timestamp": "2020-01-24T06:29:02.403Z",
"received_at": "2020-01-24T06:29:02.403Z"
},
"metadata": {
Expand Down Expand Up @@ -1106,6 +1115,9 @@ const sampleEvents = {
"loaded_at": "datetime",
"phone": "string",
"received_at": "datetime",
"sent_at": "datetime",
"timestamp": "datetime",
"original_timestamp": "datetime",
"uuid_ts": "datetime"
},
"receivedAt": "2020-01-24T11:59:02.403+05:30",
Expand Down Expand Up @@ -1204,6 +1216,9 @@ const sampleEvents = {
"EMAIL": "[email protected]",
"ID": "user123",
"PHONE": "+917836362334",
"SENT_AT": "2021-01-03T17:02:53.195Z",
"ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z",
"TIMESTAMP": "2020-01-24T06:29:02.403Z",
"RECEIVED_AT": "2020-01-24T06:29:02.403Z"
},
"metadata": {
Expand All @@ -1230,6 +1245,9 @@ const sampleEvents = {
"ID": "string",
"PHONE": "string",
"RECEIVED_AT": "datetime",
"SENT_AT": "datetime",
"TIMESTAMP": "datetime",
"ORIGINAL_TIMESTAMP": "datetime",
"UUID_TS": "datetime"
},
"receivedAt": "2020-01-24T11:59:02.403+05:30",
Expand Down Expand Up @@ -1360,6 +1378,254 @@ const sampleEvents = {
"table": "users"
}
}
],
gcs_datalake: [
{
"data": {
"timestamp": "2020-01-24T06:29:02.403Z",
"anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35",
"channel": "web",
"context_app_build": "1.0.0",
"context_app_name": "RudderLabs JavaScript SDK",
"context_app_namespace": "com.rudderlabs.javascript",
"context_app_version": "1.1.11",
"context_device_id": "id",
"context_device_token": "token",
"context_device_type": "ios",
"context_ip": "[::1]:53708",
"context_library_name": "RudderLabs JavaScript SDK",
"context_library_version": "1.1.11",
"context_locale": "en-US",
"context_os_name": "android",
"context_os_version": "1.12.3",
"context_request_ip": "[::1]:53708",
"context_traits_email": "[email protected]",
"context_traits_phone": "+917836362334",
"context_traits_user_id": "user123",
"context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0",
"email": "[email protected]",
"id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff",
"original_timestamp": "2020-01-24T06:29:02.364Z",
"phone": "+917836362334",
"received_at": "2020-01-24T06:29:02.403Z",
"sent_at": "2021-01-03T17:02:53.195Z",
"user_id": "user123"
},
"metadata": {
"columns": {
"timestamp": "datetime",
"anonymous_id": "string",
"channel": "string",
"context_app_build": "string",
"context_app_name": "string",
"context_app_namespace": "string",
"context_app_version": "string",
"context_device_id": "string",
"context_device_token": "string",
"context_device_type": "string",
"context_ip": "string",
"context_library_name": "string",
"context_library_version": "string",
"context_locale": "string",
"context_os_name": "string",
"context_os_version": "string",
"context_request_ip": "string",
"context_traits_email": "string",
"context_traits_phone": "string",
"context_traits_user_id": "string",
"context_user_agent": "string",
"email": "string",
"id": "string",
"original_timestamp": "datetime",
"phone": "string",
"received_at": "datetime",
"sent_at": "datetime",
"user_id": "string",
"uuid_ts": "datetime"
},
"receivedAt": "2020-01-24T11:59:02.403+05:30",
"table": "identifies"
}
},
{
"data": {
"context_app_build": "1.0.0",
"context_app_name": "RudderLabs JavaScript SDK",
"context_app_namespace": "com.rudderlabs.javascript",
"context_app_version": "1.1.11",
"context_device_id": "id",
"context_device_token": "token",
"context_device_type": "ios",
"context_ip": "[::1]:53708",
"context_library_name": "RudderLabs JavaScript SDK",
"context_library_version": "1.1.11",
"context_locale": "en-US",
"context_os_name": "android",
"context_os_version": "1.12.3",
"context_request_ip": "[::1]:53708",
"context_traits_email": "[email protected]",
"context_traits_phone": "+917836362334",
"context_traits_user_id": "user123",
"context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0",
"email": "[email protected]",
"id": "user123",
"phone": "+917836362334",
"received_at": "2020-01-24T06:29:02.403Z"
},
"metadata": {
"columns": {
"context_app_build": "string",
"context_app_name": "string",
"context_app_namespace": "string",
"context_app_version": "string",
"context_device_id": "string",
"context_device_token": "string",
"context_device_type": "string",
"context_ip": "string",
"context_library_name": "string",
"context_library_version": "string",
"context_locale": "string",
"context_os_name": "string",
"context_os_version": "string",
"context_request_ip": "string",
"context_traits_email": "string",
"context_traits_phone": "string",
"context_traits_user_id": "string",
"context_user_agent": "string",
"email": "string",
"id": "string",
"phone": "string",
"received_at": "datetime",
"uuid_ts": "datetime"
},
"receivedAt": "2020-01-24T11:59:02.403+05:30",
"table": "users"
}
}
],
azure_datalake: [
{
"data": {
"timestamp": "2020-01-24T06:29:02.403Z",
"anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35",
"channel": "web",
"context_app_build": "1.0.0",
"context_app_name": "RudderLabs JavaScript SDK",
"context_app_namespace": "com.rudderlabs.javascript",
"context_app_version": "1.1.11",
"context_device_id": "id",
"context_device_token": "token",
"context_device_type": "ios",
"context_ip": "[::1]:53708",
"context_library_name": "RudderLabs JavaScript SDK",
"context_library_version": "1.1.11",
"context_locale": "en-US",
"context_os_name": "android",
"context_os_version": "1.12.3",
"context_request_ip": "[::1]:53708",
"context_traits_email": "[email protected]",
"context_traits_phone": "+917836362334",
"context_traits_user_id": "user123",
"context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0",
"email": "[email protected]",
"id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff",
"original_timestamp": "2020-01-24T06:29:02.364Z",
"phone": "+917836362334",
"received_at": "2020-01-24T06:29:02.403Z",
"sent_at": "2021-01-03T17:02:53.195Z",
"user_id": "user123"
},
"metadata": {
"columns": {
"timestamp": "datetime",
"anonymous_id": "string",
"channel": "string",
"context_app_build": "string",
"context_app_name": "string",
"context_app_namespace": "string",
"context_app_version": "string",
"context_device_id": "string",
"context_device_token": "string",
"context_device_type": "string",
"context_ip": "string",
"context_library_name": "string",
"context_library_version": "string",
"context_locale": "string",
"context_os_name": "string",
"context_os_version": "string",
"context_request_ip": "string",
"context_traits_email": "string",
"context_traits_phone": "string",
"context_traits_user_id": "string",
"context_user_agent": "string",
"email": "string",
"id": "string",
"original_timestamp": "datetime",
"phone": "string",
"received_at": "datetime",
"sent_at": "datetime",
"user_id": "string",
"uuid_ts": "datetime"
},
"receivedAt": "2020-01-24T11:59:02.403+05:30",
"table": "identifies"
}
},
{
"data": {
"context_app_build": "1.0.0",
"context_app_name": "RudderLabs JavaScript SDK",
"context_app_namespace": "com.rudderlabs.javascript",
"context_app_version": "1.1.11",
"context_device_id": "id",
"context_device_token": "token",
"context_device_type": "ios",
"context_ip": "[::1]:53708",
"context_library_name": "RudderLabs JavaScript SDK",
"context_library_version": "1.1.11",
"context_locale": "en-US",
"context_os_name": "android",
"context_os_version": "1.12.3",
"context_request_ip": "[::1]:53708",
"context_traits_email": "[email protected]",
"context_traits_phone": "+917836362334",
"context_traits_user_id": "user123",
"context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0",
"email": "[email protected]",
"id": "user123",
"phone": "+917836362334",
"received_at": "2020-01-24T06:29:02.403Z"
},
"metadata": {
"columns": {
"context_app_build": "string",
"context_app_name": "string",
"context_app_namespace": "string",
"context_app_version": "string",
"context_device_id": "string",
"context_device_token": "string",
"context_device_type": "string",
"context_ip": "string",
"context_library_name": "string",
"context_library_version": "string",
"context_locale": "string",
"context_os_name": "string",
"context_os_version": "string",
"context_request_ip": "string",
"context_traits_email": "string",
"context_traits_phone": "string",
"context_traits_user_id": "string",
"context_user_agent": "string",
"email": "string",
"id": "string",
"phone": "string",
"received_at": "datetime",
"uuid_ts": "datetime"
},
"receivedAt": "2020-01-24T11:59:02.403+05:30",
"table": "users"
}
}
]
}
}
Expand All @@ -1379,6 +1645,18 @@ function opOutput(eventType, provider) {
return _.cloneDeep(sampleEvents[eventType].output.rs);
case "bq":
return _.cloneDeep(sampleEvents[eventType].output.bq);
case "gcs_datalake":
if (eventType === 'users') {
return _.cloneDeep(sampleEvents[eventType].output.gcs_datalake);
} else {
return _.cloneDeep(sampleEvents[eventType].output.default);
}
case "azure_datalake":
if (eventType === 'users') {
return _.cloneDeep(sampleEvents[eventType].output.azure_datalake);
} else {
return _.cloneDeep(sampleEvents[eventType].output.default);
}
default:
return _.cloneDeep(sampleEvents[eventType].output.default);
}
Expand Down
Loading

0 comments on commit f50effe

Please sign in to comment.