From 85f7f2dfe11398e70816d80b2b1dbfe1ab38aba1 Mon Sep 17 00:00:00 2001 From: achettyiitr Date: Thu, 5 Sep 2024 15:29:08 +0530 Subject: [PATCH] chore: properly indent events.js --- test/__tests__/data/warehouse/events.js | 4321 ++++++++++++----------- test/__tests__/warehouse.test.js | 1968 ++++++----- 2 files changed, 3203 insertions(+), 3086 deletions(-) diff --git a/test/__tests__/data/warehouse/events.js b/test/__tests__/data/warehouse/events.js index 6338dfc995..1b47ec1790 100644 --- a/test/__tests__/data/warehouse/events.js +++ b/test/__tests__/data/warehouse/events.js @@ -1,2208 +1,2209 @@ const _ = require("lodash"); +// TODO: also include events with whSchemaVersions v0 const sampleEvents = { - track: { - input: { - destination: { - Config: { - restApiKey: "dummyApiKey", - prefixProperties: true, - useNativeSDK: false, - allowUsersContextTraits: true, - allowEventContextTraits: true, - underscoreDivideNumbers: true - }, - DestinationDefinition: { - DisplayName: "Braze", - ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", - Name: "BRAZE" - }, - Enabled: true, - ID: "1WhcOCGgj9asZu850HvugU2C3Aq", - Name: "Braze", - Transformations: [] - }, - message: { - anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.5" - }, - ip: "0.0.0.0", - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.5" - }, - locale: "en-GB", - os: { - name: "", - version: "" - }, - screen: { - density: 2 - }, - traits: { - city: "Disney", - country: "USA", - email: "mickey@disney.com", - firstname: "Mickey" - }, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" - }, - event: "button clicked", - integrations: { - All: true - }, - messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - originalTimestamp: "2020-01-24T06:29:02.364Z", - properties: { - currency: "USD", - revenue: 50, - stack: { - history: { - errorDetails: [ - { - "message": "Cannot set headers after they are sent to the client", - "toString": "[function]" - } - ] - } - } - }, - receivedAt: "2020-01-24T11:59:02.403+05:30", - request_ip: "[::1]:53708", - sentAt: "2020-01-24T06:29:02.364Z", - timestamp: "2020-01-24T11:59:02.403+05:30", - type: "track", - userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" - }, - request: { - query: { - whSchemaVersion: "v1" - } - } + track: { + input: { + destination: { + Config: { + restApiKey: "dummyApiKey", + prefixProperties: true, + useNativeSDK: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true }, - output: { - default: [ - { - metadata: { - table: "tracks", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - event_text: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - event: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - event_text: "button clicked", - id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.364Z", - timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.364Z", - channel: "web", - event: "button_clicked" - } - }, - { - metadata: { - table: "button_clicked", - columns: { - uuid_ts: "datetime", - currency: "string", - revenue: "int", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - event_text: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - event: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - currency: "USD", - revenue: 50, - stack_history_error_details: [ - { - "message": "Cannot set headers after they are sent to the client", - "toString": "[function]" - } - ], - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - event_text: "button clicked", - id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.364Z", - timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.364Z", - channel: "web", - event: "button_clicked" - } - } - ], - snowflake: [ - { - metadata: { - table: "TRACKS", - columns: { - UUID_TS: "datetime", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_TRAITS_CITY: "string", - CONTEXT_TRAITS_COUNTRY: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_TRAITS_FIRSTNAME: "string", - CONTEXT_USER_AGENT: "string", - EVENT_TEXT: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - CHANNEL: "string", - EVENT: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_TRAITS_CITY: "Disney", - CONTEXT_TRAITS_COUNTRY: "USA", - CONTEXT_TRAITS_EMAIL: "mickey@disney.com", - CONTEXT_TRAITS_FIRSTNAME: "Mickey", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - EVENT_TEXT: "button clicked", - ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - SENT_AT: "2020-01-24T06:29:02.364Z", - TIMESTAMP: "2020-01-24T06:29:02.403Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", - CHANNEL: "web", - EVENT: "button_clicked" - } - }, - { - metadata: { - table: "BUTTON_CLICKED", - columns: { - UUID_TS: "datetime", - CURRENCY: "string", - REVENUE: "int", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_TRAITS_CITY: "string", - CONTEXT_TRAITS_COUNTRY: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_TRAITS_FIRSTNAME: "string", - CONTEXT_USER_AGENT: "string", - EVENT_TEXT: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - CHANNEL: "string", - EVENT: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - CURRENCY: "USD", - REVENUE: 50, - STACK_HISTORY_ERROR_DETAILS: [ - { - "message": "Cannot set headers after they are sent to the client", - "toString": "[function]" - } - ], - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_TRAITS_CITY: "Disney", - CONTEXT_TRAITS_COUNTRY: "USA", - CONTEXT_TRAITS_EMAIL: "mickey@disney.com", - CONTEXT_TRAITS_FIRSTNAME: "Mickey", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - EVENT_TEXT: "button clicked", - ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - SENT_AT: "2020-01-24T06:29:02.364Z", - TIMESTAMP: "2020-01-24T06:29:02.403Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", - CHANNEL: "web", - EVENT: "button_clicked", - } - } - ], - s3_datalake: [ - { - metadata: { - table: "tracks", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - event_text: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - _timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - event: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - event_text: "button clicked", - id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.364Z", - _timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.364Z", - channel: "web", - event: "button_clicked" - } - }, + DestinationDefinition: { + DisplayName: "Braze", + ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", + Name: "BRAZE" + }, + Enabled: true, + ID: "1WhcOCGgj9asZu850HvugU2C3Aq", + Name: "Braze", + Transformations: [] + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + os: { + name: "", + version: "" + }, + screen: { + density: 2 + }, + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + event: "button clicked", + integrations: { + All: true + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + properties: { + currency: "USD", + revenue: 50, + stack: { + history: { + errorDetails: [ { - metadata: { - table: "button_clicked", - columns: { - uuid_ts: "datetime", - currency: "string", - revenue: "int", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - event_text: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - _timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - event: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - currency: "USD", - revenue: 50, - stack_history_error_details: [ - { - "message": "Cannot set headers after they are sent to the client", - "toString": "[function]" - } - ], - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - event_text: "button clicked", - id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.364Z", - _timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.364Z", - channel: "web", - event: "button_clicked" - } + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" } - ] + ] + } + } + }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "track", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" } + } }, - identify: { - input: { - destination: { - Config: { - restApiKey: "dummyApiKey", - prefixProperties: true, - useNativeSDK: false, - allowUsersContextTraits: true, - allowEventContextTraits: true, - underscoreDivideNumbers: true - }, - DestinationDefinition: { - DisplayName: "Braze", - ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", - Name: "BRAZE" - }, - Enabled: true, - ID: "1WhcOCGgj9asZu850HvugU2C3Aq", - Name: "Braze", - Transformations: [] + output: { + default: [ + { + metadata: { + table: "tracks", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + event_text: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + event: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + event_text: "button clicked", + id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.364Z", + channel: "web", + event: "button_clicked" + } + }, + { + metadata: { + table: "button_clicked", + columns: { + uuid_ts: "datetime", + currency: "string", + revenue: "int", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + event_text: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + event: "string" }, - message: { - anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.5" - }, - ip: "0.0.0.0", - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.5" - }, - locale: "en-GB", - os: { - name: "", - version: "" - }, - screen: { - density: 2 - }, - traits: { - city: "Disney", - country: "USA", - email: "mickey@disney.com", - firstname: "Mickey" - }, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" - }, - traits: { - country: "UK", - lastname: "Mouse" - }, - integrations: { - All: true - }, - messageId: "2536eda4-d638-4c93-8014-8ffe3f083214", - originalTimestamp: "2020-01-24T06:29:02.362Z", - receivedAt: "2020-01-24T11:59:02.403+05:30", - request_ip: "[::1]:53708", - sentAt: "2020-01-24T06:29:02.363Z", - timestamp: "2020-01-24T11:59:02.402+05:30", - type: "identify", - userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + currency: "USD", + revenue: 50, + stack_history_error_details: [ + { + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" + } + ], + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + event_text: "button clicked", + id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.364Z", + channel: "web", + event: "button_clicked" + } + } + ], + snowflake: [ + { + metadata: { + table: "TRACKS", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + EVENT: "string" }, - request: { - query: { - whSchemaVersion: "v1" - } - } + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "button clicked", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + EVENT: "button_clicked" + } }, - output: { - default: [ - { - metadata: { - table: "identifies", - columns: { - uuid_ts: "datetime", - city: "string", - country: "string", - email: "string", - firstname: "string", - lastname: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - city: "Disney", - country: "UK", - email: "mickey@disney.com", - firstname: "Mickey", - lastname: "Mouse", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "2536eda4-d638-4c93-8014-8ffe3f083214", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.363Z", - timestamp: "2020-01-24T06:29:02.402Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.362Z", - channel: "web" - } - }, - { - metadata: { - table: "users", - columns: { - uuid_ts: "datetime", - city: "string", - country: "string", - email: "string", - firstname: "string", - lastname: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - id: "string", - received_at: "datetime" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - city: "Disney", - country: "UK", - email: "mickey@disney.com", - firstname: "Mickey", - lastname: "Mouse", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - received_at: "2020-01-24T06:29:02.403Z" - } - } + { + metadata: { + table: "BUTTON_CLICKED", + columns: { + UUID_TS: "datetime", + CURRENCY: "string", + REVENUE: "int", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CURRENCY: "USD", + REVENUE: 50, + STACK_HISTORY_ERROR_DETAILS: [ + { + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" + } ], - snowflake: [ - { - metadata: { - table: "IDENTIFIES", - columns: { - UUID_TS: "datetime", - CITY: "string", - COUNTRY: "string", - EMAIL: "string", - FIRSTNAME: "string", - LASTNAME: "string", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_TRAITS_CITY: "string", - CONTEXT_TRAITS_COUNTRY: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_TRAITS_FIRSTNAME: "string", - CONTEXT_USER_AGENT: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - CHANNEL: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - CITY: "Disney", - COUNTRY: "UK", - EMAIL: "mickey@disney.com", - FIRSTNAME: "Mickey", - LASTNAME: "Mouse", - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_TRAITS_CITY: "Disney", - CONTEXT_TRAITS_COUNTRY: "USA", - CONTEXT_TRAITS_EMAIL: "mickey@disney.com", - CONTEXT_TRAITS_FIRSTNAME: "Mickey", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - ID: "2536eda4-d638-4c93-8014-8ffe3f083214", - ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - SENT_AT: "2020-01-24T06:29:02.363Z", - TIMESTAMP: "2020-01-24T06:29:02.402Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.362Z", - CHANNEL: "web" - } - }, - { - metadata: { - table: "USERS", - columns: { - UUID_TS: "datetime", - CITY: "string", - COUNTRY: "string", - EMAIL: "string", - FIRSTNAME: "string", - LASTNAME: "string", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_TRAITS_CITY: "string", - CONTEXT_TRAITS_COUNTRY: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_TRAITS_FIRSTNAME: "string", - CONTEXT_USER_AGENT: "string", - ID: "string", - RECEIVED_AT: "datetime" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - CITY: "Disney", - COUNTRY: "UK", - EMAIL: "mickey@disney.com", - FIRSTNAME: "Mickey", - LASTNAME: "Mouse", - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_TRAITS_CITY: "Disney", - CONTEXT_TRAITS_COUNTRY: "USA", - CONTEXT_TRAITS_EMAIL: "mickey@disney.com", - CONTEXT_TRAITS_FIRSTNAME: "Mickey", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - RECEIVED_AT: "2020-01-24T06:29:02.403Z" - } - } + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "button clicked", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + EVENT: "button_clicked", + } + } + ], + s3_datalake: [ + { + metadata: { + table: "tracks", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + event_text: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + _timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + event: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + event_text: "button clicked", + id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.364Z", + _timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.364Z", + channel: "web", + event: "button_clicked" + } + }, + { + metadata: { + table: "button_clicked", + columns: { + uuid_ts: "datetime", + currency: "string", + revenue: "int", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + event_text: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + _timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + event: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + currency: "USD", + revenue: 50, + stack_history_error_details: [ + { + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" + } ], - s3_datalake: [ - { - metadata: { - table: "identifies", - columns: { - uuid_ts: "datetime", - city: "string", - country: "string", - email: "string", - firstname: "string", - lastname: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - _timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - city: "Disney", - country: "UK", - email: "mickey@disney.com", - firstname: "Mickey", - lastname: "Mouse", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "2536eda4-d638-4c93-8014-8ffe3f083214", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.363Z", - _timestamp: "2020-01-24T06:29:02.402Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.362Z", - channel: "web" - } - }, - { - metadata: { - table: "users", - columns: { - uuid_ts: "datetime", - city: "string", - country: "string", - email: "string", - firstname: "string", - lastname: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - id: "string", - received_at: "datetime" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - city: "Disney", - country: "UK", - email: "mickey@disney.com", - firstname: "Mickey", - lastname: "Mouse", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - received_at: "2020-01-24T06:29:02.403Z" - } - } - ] + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + event_text: "button clicked", + id: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.364Z", + _timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.364Z", + channel: "web", + event: "button_clicked" + } } + ] + } + }, + identify: { + input: { + destination: { + Config: { + restApiKey: "dummyApiKey", + prefixProperties: true, + useNativeSDK: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + DestinationDefinition: { + DisplayName: "Braze", + ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", + Name: "BRAZE" + }, + Enabled: true, + ID: "1WhcOCGgj9asZu850HvugU2C3Aq", + Name: "Braze", + Transformations: [] + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + os: { + name: "", + version: "" + }, + screen: { + density: 2 + }, + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + traits: { + country: "UK", + lastname: "Mouse" + }, + integrations: { + All: true + }, + messageId: "2536eda4-d638-4c93-8014-8ffe3f083214", + originalTimestamp: "2020-01-24T06:29:02.362Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.363Z", + timestamp: "2020-01-24T11:59:02.402+05:30", + type: "identify", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } }, - page: { - input: { - message: { - anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.5" - }, - ip: "0.0.0.0", - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.5" - }, - locale: "en-GB", - os: { - name: "", - version: "" - }, - screen: { - density: 2 - }, - traits: {}, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" - }, - integrations: { - All: true - }, - messageId: "dd266c67-9199-4a52-ba32-f46ddde67312", - originalTimestamp: "2020-01-24T06:29:02.358Z", - properties: { - path: "/tests/html/index2.html", - referrer: "", - search: "", - title: "", - url: "http://localhost/tests/html/index2.html" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30", - request_ip: "[::1]:53708", - sentAt: "2020-01-24T06:29:02.359Z", - timestamp: "2020-01-24T11:59:02.402+05:30", - type: "page", - name: "sample title", - userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + output: { + default: [ + { + metadata: { + table: "identifies", + columns: { + uuid_ts: "datetime", + city: "string", + country: "string", + email: "string", + firstname: "string", + lastname: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string" }, - destination: { - Config: { - restApiKey: "dummyApiKey", - prefixProperties: true, - useNativeSDK: false, - allowUsersContextTraits: true, - allowEventContextTraits: true, - underscoreDivideNumbers: true - }, - DestinationDefinition: { - DisplayName: "Braze", - ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", - Name: "BRAZE" - }, - Enabled: true, - ID: "1WhcOCGgj9asZu850HvugU2C3Aq", - Name: "Braze", - Transformations: [] + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + city: "Disney", + country: "UK", + email: "mickey@disney.com", + firstname: "Mickey", + lastname: "Mouse", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "2536eda4-d638-4c93-8014-8ffe3f083214", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.363Z", + timestamp: "2020-01-24T06:29:02.402Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.362Z", + channel: "web" + } + }, + { + metadata: { + table: "users", + columns: { + uuid_ts: "datetime", + city: "string", + country: "string", + email: "string", + firstname: "string", + lastname: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + id: "string", + received_at: "datetime" }, - request: { - query: { - whSchemaVersion: "v1" - } - } + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + city: "Disney", + country: "UK", + email: "mickey@disney.com", + firstname: "Mickey", + lastname: "Mouse", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + received_at: "2020-01-24T06:29:02.403Z" + } + } + ], + snowflake: [ + { + metadata: { + table: "IDENTIFIES", + columns: { + UUID_TS: "datetime", + CITY: "string", + COUNTRY: "string", + EMAIL: "string", + FIRSTNAME: "string", + LASTNAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CITY: "Disney", + COUNTRY: "UK", + EMAIL: "mickey@disney.com", + FIRSTNAME: "Mickey", + LASTNAME: "Mouse", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "2536eda4-d638-4c93-8014-8ffe3f083214", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.363Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.362Z", + CHANNEL: "web" + } }, - output: { - default: [ - { - metadata: { - table: "pages", - columns: { - uuid_ts: "datetime", - path: "string", - url: "string", - name: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_user_agent: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - path: "/tests/html/index2.html", - url: "http://localhost/tests/html/index2.html", - name: "sample title", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "dd266c67-9199-4a52-ba32-f46ddde67312", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.359Z", - timestamp: "2020-01-24T06:29:02.402Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.358Z", - channel: "web" - } - } - ], - snowflake: [ - { - metadata: { - table: "PAGES", - columns: { - UUID_TS: "datetime", - PATH: "string", - URL: "string", - NAME: "string", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_USER_AGENT: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - CHANNEL: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - PATH: "/tests/html/index2.html", - URL: "http://localhost/tests/html/index2.html", - NAME: "sample title", - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - ID: "dd266c67-9199-4a52-ba32-f46ddde67312", - ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - SENT_AT: "2020-01-24T06:29:02.359Z", - TIMESTAMP: "2020-01-24T06:29:02.402Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.358Z", - CHANNEL: "web" - } - } - ], - s3_datalake: [ - { - metadata: { - table: "pages", - columns: { - uuid_ts: "datetime", - path: "string", - url: "string", - name: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_user_agent: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - _timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - path: "/tests/html/index2.html", - url: "http://localhost/tests/html/index2.html", - name: "sample title", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "dd266c67-9199-4a52-ba32-f46ddde67312", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2020-01-24T06:29:02.359Z", - _timestamp: "2020-01-24T06:29:02.402Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.358Z", - channel: "web" - } - } - ] + { + metadata: { + table: "USERS", + columns: { + UUID_TS: "datetime", + CITY: "string", + COUNTRY: "string", + EMAIL: "string", + FIRSTNAME: "string", + LASTNAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + ID: "string", + RECEIVED_AT: "datetime" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CITY: "Disney", + COUNTRY: "UK", + EMAIL: "mickey@disney.com", + FIRSTNAME: "Mickey", + LASTNAME: "Mouse", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + RECEIVED_AT: "2020-01-24T06:29:02.403Z" + } } - }, - screen: { - input: { - message: { - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.0" - }, - traits: { - email: "test@rudderstack.com" - }, - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.0" - }, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - locale: "en-US", - ip: "0.0.0.0", - os: { - name: "", - version: "" - }, - screen: { - density: 2 - } - }, - type: "screen", - messageId: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", - originalTimestamp: "2019-10-14T11:15:18.299Z", - receivedAt: "2020-01-24T11:59:02.403+05:30", - timestamp: "2020-01-24T11:59:02.402+05:30", - anonymousId: "00000000000000000000000000", - userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - request_ip: "[::1]:53708", - properties: { - path: "/abc", - referrer: "", - search: "", - title: "", - url: "", - category: "test-category" - }, - integrations: { - All: true - }, - name: "ApplicationLoaded", - sentAt: "2019-10-14T11:15:53.296Z" + ], + s3_datalake: [ + { + metadata: { + table: "identifies", + columns: { + uuid_ts: "datetime", + city: "string", + country: "string", + email: "string", + firstname: "string", + lastname: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + _timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string" }, - destination: { - Config: { - apiKey: "12345", - mapToSingleEvent: false, - trackAllPages: false, - trackCategorisedPages: true, - trackNamedPages: false, - allowUsersContextTraits: true, - allowEventContextTraits: true, - underscoreDivideNumbers: true - }, - Enabled: true + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + city: "Disney", + country: "UK", + email: "mickey@disney.com", + firstname: "Mickey", + lastname: "Mouse", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "2536eda4-d638-4c93-8014-8ffe3f083214", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.363Z", + _timestamp: "2020-01-24T06:29:02.402Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.362Z", + channel: "web" + } + }, + { + metadata: { + table: "users", + columns: { + uuid_ts: "datetime", + city: "string", + country: "string", + email: "string", + firstname: "string", + lastname: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + id: "string", + received_at: "datetime" }, - request: { - query: { - whSchemaVersion: "v1" - } - } + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + city: "Disney", + country: "UK", + email: "mickey@disney.com", + firstname: "Mickey", + lastname: "Mouse", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + received_at: "2020-01-24T06:29:02.403Z" + } + } + ] + } + }, + page: { + input: { + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + os: { + name: "", + version: "" + }, + screen: { + density: 2 + }, + traits: {}, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" }, - output: { - default: [ - { - metadata: { - table: "screens", - columns: { - uuid_ts: "datetime", - path: "string", - category: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_traits_email: "string", - context_library_name: "string", - context_library_version: "string", - context_user_agent: "string", - context_locale: "string", - context_screen_density: "int", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - original_timestamp: "datetime", - received_at: "datetime", - timestamp: "datetime", - channel: "string", - name: "string" - } - }, - data: { - path: "/abc", - category: "test-category", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.0", - context_traits_email: "test@rudderstack.com", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.0", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - context_locale: "en-US", - context_screen_density: 2, - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - id: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", - anonymous_id: "00000000000000000000000000", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2019-10-14T11:15:53.296Z", - original_timestamp: "2019-10-14T11:15:18.299Z", - received_at: "2020-01-24T06:29:02.403Z", - timestamp: "2020-01-24T06:29:02.402Z", - channel: "web", - name: "ApplicationLoaded" - } - } - ], - snowflake: [ - { - metadata: { - table: "SCREENS", - columns: { - UUID_TS: "datetime", - PATH: "string", - CATEGORY: "string", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_USER_AGENT: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - TIMESTAMP: "datetime", - CHANNEL: "string", - NAME: "string" - } - }, - data: { - PATH: "/abc", - CATEGORY: "test-category", - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.0", - CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.0", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - CONTEXT_LOCALE: "en-US", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - ID: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", - ANONYMOUS_ID: "00000000000000000000000000", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - SENT_AT: "2019-10-14T11:15:53.296Z", - ORIGINAL_TIMESTAMP: "2019-10-14T11:15:18.299Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - TIMESTAMP: "2020-01-24T06:29:02.402Z", - CHANNEL: "web", - NAME: "ApplicationLoaded" - } - } - ], - s3_datalake: [ - { - metadata: { - table: "screens", - columns: { - uuid_ts: "datetime", - path: "string", - category: "string", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_traits_email: "string", - context_library_name: "string", - context_library_version: "string", - context_user_agent: "string", - context_locale: "string", - context_screen_density: "int", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - original_timestamp: "datetime", - received_at: "datetime", - _timestamp: "datetime", - channel: "string", - name: "string" - } - }, - data: { - path: "/abc", - category: "test-category", - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.0", - context_traits_email: "test@rudderstack.com", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.0", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - context_locale: "en-US", - context_screen_density: 2, - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - id: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", - anonymous_id: "00000000000000000000000000", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - sent_at: "2019-10-14T11:15:53.296Z", - original_timestamp: "2019-10-14T11:15:18.299Z", - received_at: "2020-01-24T06:29:02.403Z", - _timestamp: "2020-01-24T06:29:02.402Z", - channel: "web", - name: "ApplicationLoaded" - } - } - ] + integrations: { + All: true + }, + messageId: "dd266c67-9199-4a52-ba32-f46ddde67312", + originalTimestamp: "2020-01-24T06:29:02.358Z", + properties: { + path: "/tests/html/index2.html", + referrer: "", + search: "", + title: "", + url: "http://localhost/tests/html/index2.html" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.359Z", + timestamp: "2020-01-24T11:59:02.402+05:30", + type: "page", + name: "sample title", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + destination: { + Config: { + restApiKey: "dummyApiKey", + prefixProperties: true, + useNativeSDK: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + DestinationDefinition: { + DisplayName: "Braze", + ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", + Name: "BRAZE" + }, + Enabled: true, + ID: "1WhcOCGgj9asZu850HvugU2C3Aq", + Name: "Braze", + Transformations: [] + }, + request: { + query: { + whSchemaVersion: "v1" } + } }, - group: { - input: { - message: { - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.0" - }, - traits: { - email: "test@rudderstack.com" - }, - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.0" - }, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - locale: "en-US", - ip: "0.0.0.0", - os: { - id: "72e528f869711c3d", - manufacturer: "Google", - model: "sdk_gphone_x86", - name: "generic_x86_arm", - token: "some_device_token", - type: "android" - }, - screen: { - density: 2 - } - }, - type: "group", - messageId: "84e26acc-56a5-4835-8233-591137fca468", - originalTimestamp: "2019-10-14T09:03:17.562Z", - timestamp: "2020-01-24T11:59:02.403+05:30", - receivedAt: "2020-01-24T11:59:02.403+05:30", - anonymousId: "00000000000000000000000000", - userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - groupId: "g1", - integrations: { - All: true - }, - sentAt: "2019-10-14T09:03:22.563Z" + output: { + default: [ + { + metadata: { + table: "pages", + columns: { + uuid_ts: "datetime", + path: "string", + url: "string", + name: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_user_agent: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string" }, - destination: { - Config: { - apiKey: "12345", - mapToSingleEvent: false, - trackAllPages: true, - trackCategorisedPages: false, - trackNamedPages: false, - allowUsersContextTraits: true, - allowEventContextTraits: true, - underscoreDivideNumbers: true - }, - Enabled: true + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + path: "/tests/html/index2.html", + url: "http://localhost/tests/html/index2.html", + name: "sample title", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "dd266c67-9199-4a52-ba32-f46ddde67312", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.359Z", + timestamp: "2020-01-24T06:29:02.402Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.358Z", + channel: "web" + } + } + ], + snowflake: [ + { + metadata: { + table: "PAGES", + columns: { + UUID_TS: "datetime", + PATH: "string", + URL: "string", + NAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_USER_AGENT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string" }, - request: { - query: { - whSchemaVersion: "v1" - } + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + PATH: "/tests/html/index2.html", + URL: "http://localhost/tests/html/index2.html", + NAME: "sample title", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "dd266c67-9199-4a52-ba32-f46ddde67312", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.359Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.358Z", + CHANNEL: "web" + } + } + ], + s3_datalake: [ + { + metadata: { + table: "pages", + columns: { + uuid_ts: "datetime", + path: "string", + url: "string", + name: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_user_agent: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + _timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + path: "/tests/html/index2.html", + url: "http://localhost/tests/html/index2.html", + name: "sample title", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "dd266c67-9199-4a52-ba32-f46ddde67312", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2020-01-24T06:29:02.359Z", + _timestamp: "2020-01-24T06:29:02.402Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.358Z", + channel: "web" + } + } + ] + } + }, + screen: { + input: { + message: { + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.0" + }, + traits: { + email: "test@rudderstack.com" + }, + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.0" + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + locale: "en-US", + ip: "0.0.0.0", + os: { + name: "", + version: "" + }, + screen: { + density: 2 + } + }, + type: "screen", + messageId: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + originalTimestamp: "2019-10-14T11:15:18.299Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + timestamp: "2020-01-24T11:59:02.402+05:30", + anonymousId: "00000000000000000000000000", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + request_ip: "[::1]:53708", + properties: { + path: "/abc", + referrer: "", + search: "", + title: "", + url: "", + category: "test-category" + }, + integrations: { + All: true + }, + name: "ApplicationLoaded", + sentAt: "2019-10-14T11:15:53.296Z" + }, + destination: { + Config: { + apiKey: "12345", + mapToSingleEvent: false, + trackAllPages: false, + trackCategorisedPages: true, + trackNamedPages: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + Enabled: true + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + metadata: { + table: "screens", + columns: { + uuid_ts: "datetime", + path: "string", + category: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_traits_email: "string", + context_library_name: "string", + context_library_version: "string", + context_user_agent: "string", + context_locale: "string", + context_screen_density: "int", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + original_timestamp: "datetime", + received_at: "datetime", + timestamp: "datetime", + channel: "string", + name: "string" + } + }, + data: { + path: "/abc", + category: "test-category", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.0", + context_traits_email: "test@rudderstack.com", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.0", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + context_locale: "en-US", + context_screen_density: 2, + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + id: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + anonymous_id: "00000000000000000000000000", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2019-10-14T11:15:53.296Z", + original_timestamp: "2019-10-14T11:15:18.299Z", + received_at: "2020-01-24T06:29:02.403Z", + timestamp: "2020-01-24T06:29:02.402Z", + channel: "web", + name: "ApplicationLoaded" + } + } + ], + snowflake: [ + { + metadata: { + table: "SCREENS", + columns: { + UUID_TS: "datetime", + PATH: "string", + CATEGORY: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + TIMESTAMP: "datetime", + CHANNEL: "string", + NAME: "string" } + }, + data: { + PATH: "/abc", + CATEGORY: "test-category", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.0", + CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.0", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + CONTEXT_LOCALE: "en-US", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + ID: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + ANONYMOUS_ID: "00000000000000000000000000", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2019-10-14T11:15:53.296Z", + ORIGINAL_TIMESTAMP: "2019-10-14T11:15:18.299Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + CHANNEL: "web", + NAME: "ApplicationLoaded" + } + } + ], + s3_datalake: [ + { + metadata: { + table: "screens", + columns: { + uuid_ts: "datetime", + path: "string", + category: "string", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_traits_email: "string", + context_library_name: "string", + context_library_version: "string", + context_user_agent: "string", + context_locale: "string", + context_screen_density: "int", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + original_timestamp: "datetime", + received_at: "datetime", + _timestamp: "datetime", + channel: "string", + name: "string" + } + }, + data: { + path: "/abc", + category: "test-category", + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.0", + context_traits_email: "test@rudderstack.com", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.0", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + context_locale: "en-US", + context_screen_density: 2, + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + id: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + anonymous_id: "00000000000000000000000000", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + sent_at: "2019-10-14T11:15:53.296Z", + original_timestamp: "2019-10-14T11:15:18.299Z", + received_at: "2020-01-24T06:29:02.403Z", + _timestamp: "2020-01-24T06:29:02.402Z", + channel: "web", + name: "ApplicationLoaded" + } + } + ] + } + }, + group: { + input: { + message: { + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.0" + }, + traits: { + email: "test@rudderstack.com" + }, + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.0" + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + locale: "en-US", + ip: "0.0.0.0", + os: { + id: "72e528f869711c3d", + manufacturer: "Google", + model: "sdk_gphone_x86", + name: "generic_x86_arm", + token: "some_device_token", + type: "android" + }, + screen: { + density: 2 + } }, - output: { - default: [ - { - metadata: { - table: "groups", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_traits_email: "string", - context_library_name: "string", - context_library_version: "string", - context_user_agent: "string", - context_locale: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_os_id: "string", - context_os_manufacturer: "string", - context_os_model: "string", - context_os_type: "string", - context_screen_density: "int", - id: "string", - anonymous_id: "string", - user_id: "string", - group_id: "string", - sent_at: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - timestamp: "datetime", - channel: "string" - } - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.0", - context_traits_email: "test@rudderstack.com", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.0", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - context_locale: "en-US", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_os_id: "72e528f869711c3d", - context_os_manufacturer: "Google", - context_os_model: "sdk_gphone_x86", - context_os_name: "", - context_os_token: "", - context_os_type: "android", - context_screen_density: 2, - id: "84e26acc-56a5-4835-8233-591137fca468", - anonymous_id: "00000000000000000000000000", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - group_id: "g1", - sent_at: "2019-10-14T09:03:22.563Z", - original_timestamp: "2019-10-14T09:03:17.562Z", - timestamp: "2020-01-24T11:59:02.403Z", - received_at: "2020-01-24T11:59:02.403Z", - channel: "web" - } - } - ], - snowflake: [ - { - metadata: { - table: "GROUPS", - columns: { - UUID_TS: "datetime", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_USER_AGENT: "string", - CONTEXT_LOCALE: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - CONTEXT_OS_ID: "string", - CONTEXT_OS_MANUFACTURER: "string", - CONTEXT_OS_MODEL: "string", - CONTEXT_OS_TYPE: "string", - CONTEXT_SCREEN_DENSITY: "int", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - GROUP_ID: "string", - SENT_AT: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - TIMESTAMP: "datetime", - CHANNEL: "string" - } - }, - data: { - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.0", - CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.0", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - CONTEXT_LOCALE: "en-US", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_OS_ID: "72e528f869711c3d", - CONTEXT_OS_MANUFACTURER: "Google", - CONTEXT_OS_MODEL: "sdk_gphone_x86", - CONTEXT_OS_NAME: "", - CONTEXT_OS_TOKEN: "", - CONTEXT_OS_TYPE: "android", - CONTEXT_SCREEN_DENSITY: 2, - ID: "84e26acc-56a5-4835-8233-591137fca468", - ANONYMOUS_ID: "00000000000000000000000000", - USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - GROUP_ID: "g1", - SENT_AT: "2019-10-14T09:03:22.563Z", - ORIGINAL_TIMESTAMP: "2019-10-14T09:03:17.562Z", - TIMESTAMP: "2020-01-24T11:59:02.403Z", - RECEIVED_AT: "2020-01-24T11:59:02.403Z", - CHANNEL: "web" - } - } - ], - s3_datalake: [ - { - metadata: { - table: "groups", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_traits_email: "string", - context_library_name: "string", - context_library_version: "string", - context_user_agent: "string", - context_locale: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - context_os_id: "string", - context_os_manufacturer: "string", - context_os_model: "string", - context_os_type: "string", - context_screen_density: "int", - id: "string", - anonymous_id: "string", - user_id: "string", - group_id: "string", - sent_at: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - _timestamp: "datetime", - channel: "string" - } - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.0", - context_traits_email: "test@rudderstack.com", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.0", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", - context_locale: "en-US", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_os_id: "72e528f869711c3d", - context_os_manufacturer: "Google", - context_os_model: "sdk_gphone_x86", - context_os_name: "", - context_os_token: "", - context_os_type: "android", - context_screen_density: 2, - id: "84e26acc-56a5-4835-8233-591137fca468", - anonymous_id: "00000000000000000000000000", - user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", - group_id: "g1", - sent_at: "2019-10-14T09:03:22.563Z", - original_timestamp: "2019-10-14T09:03:17.562Z", - _timestamp: "2020-01-24T11:59:02.403Z", - received_at: "2020-01-24T11:59:02.403Z", - channel: "web" - } - } - ] + type: "group", + messageId: "84e26acc-56a5-4835-8233-591137fca468", + originalTimestamp: "2019-10-14T09:03:17.562Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + receivedAt: "2020-01-24T11:59:02.403+05:30", + anonymousId: "00000000000000000000000000", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + groupId: "g1", + integrations: { + All: true + }, + sentAt: "2019-10-14T09:03:22.563Z" + }, + destination: { + Config: { + apiKey: "12345", + mapToSingleEvent: false, + trackAllPages: true, + trackCategorisedPages: false, + trackNamedPages: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true + }, + Enabled: true + }, + request: { + query: { + whSchemaVersion: "v1" } + } }, - alias: { - input: { - destination: { - Config: { - apiKey: "dummyApiKey", - prefixProperties: true, - useNativeSDK: false, - allowUsersContextTraits: true, - allowEventContextTraits: true, - underscoreDivideNumbers: true - }, - DestinationDefinition: { - DisplayName: "Kiss Metrics", - ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", - Name: "KISSMETRICS" - }, - Enabled: true, - ID: "1WhcOCGgj9asZu850HvugU2C3Aq", - Name: "Kiss Metrics", - Transformations: [] - }, - message: { - anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - channel: "web", - context: { - app: { - build: "1.0.0", - name: "RudderLabs JavaScript SDK", - namespace: "com.rudderlabs.javascript", - version: "1.0.5" - }, - ip: "0.0.0.0", - library: { - name: "RudderLabs JavaScript SDK", - version: "1.0.5" - }, - locale: "en-GB", - os: { - name: "", - version: "" - }, - screen: { - density: 2 - }, - traits: { - city: "Disney", - country: "USA", - email: "mickey@disney.com", - firstname: "Mickey" - }, - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" - }, - integrations: { - All: true - }, - messageId: "79313729-7fe5-4204-963a-dc46f4205e4e", - originalTimestamp: "2020-01-24T06:29:02.366Z", - previousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - receivedAt: "2020-01-24T11:59:02.403+05:30", - request_ip: "[::1]:53708", - sentAt: "2020-01-24T06:29:02.366Z", - timestamp: "2020-01-24T11:59:02.403+05:30", - type: "alias", - userId: "1234abc" - }, - request: { - query: { - whSchemaVersion: "v1" - } + output: { + default: [ + { + metadata: { + table: "groups", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_traits_email: "string", + context_library_name: "string", + context_library_version: "string", + context_user_agent: "string", + context_locale: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_os_id: "string", + context_os_manufacturer: "string", + context_os_model: "string", + context_os_type: "string", + context_screen_density: "int", + id: "string", + anonymous_id: "string", + user_id: "string", + group_id: "string", + sent_at: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + timestamp: "datetime", + channel: "string" + } + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.0", + context_traits_email: "test@rudderstack.com", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.0", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + context_locale: "en-US", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_os_id: "72e528f869711c3d", + context_os_manufacturer: "Google", + context_os_model: "sdk_gphone_x86", + context_os_name: "", + context_os_token: "", + context_os_type: "android", + context_screen_density: 2, + id: "84e26acc-56a5-4835-8233-591137fca468", + anonymous_id: "00000000000000000000000000", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + group_id: "g1", + sent_at: "2019-10-14T09:03:22.563Z", + original_timestamp: "2019-10-14T09:03:17.562Z", + timestamp: "2020-01-24T11:59:02.403Z", + received_at: "2020-01-24T11:59:02.403Z", + channel: "web" + } + } + ], + snowflake: [ + { + metadata: { + table: "GROUPS", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_LOCALE: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_OS_ID: "string", + CONTEXT_OS_MANUFACTURER: "string", + CONTEXT_OS_MODEL: "string", + CONTEXT_OS_TYPE: "string", + CONTEXT_SCREEN_DENSITY: "int", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + GROUP_ID: "string", + SENT_AT: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + TIMESTAMP: "datetime", + CHANNEL: "string" } + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.0", + CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.0", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + CONTEXT_LOCALE: "en-US", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_OS_ID: "72e528f869711c3d", + CONTEXT_OS_MANUFACTURER: "Google", + CONTEXT_OS_MODEL: "sdk_gphone_x86", + CONTEXT_OS_NAME: "", + CONTEXT_OS_TOKEN: "", + CONTEXT_OS_TYPE: "android", + CONTEXT_SCREEN_DENSITY: 2, + ID: "84e26acc-56a5-4835-8233-591137fca468", + ANONYMOUS_ID: "00000000000000000000000000", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + GROUP_ID: "g1", + SENT_AT: "2019-10-14T09:03:22.563Z", + ORIGINAL_TIMESTAMP: "2019-10-14T09:03:17.562Z", + TIMESTAMP: "2020-01-24T11:59:02.403Z", + RECEIVED_AT: "2020-01-24T11:59:02.403Z", + CHANNEL: "web" + } + } + ], + s3_datalake: [ + { + metadata: { + table: "groups", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_traits_email: "string", + context_library_name: "string", + context_library_version: "string", + context_user_agent: "string", + context_locale: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + context_os_id: "string", + context_os_manufacturer: "string", + context_os_model: "string", + context_os_type: "string", + context_screen_density: "int", + id: "string", + anonymous_id: "string", + user_id: "string", + group_id: "string", + sent_at: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + _timestamp: "datetime", + channel: "string" + } + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.0", + context_traits_email: "test@rudderstack.com", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.0", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + context_locale: "en-US", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_os_id: "72e528f869711c3d", + context_os_manufacturer: "Google", + context_os_model: "sdk_gphone_x86", + context_os_name: "", + context_os_token: "", + context_os_type: "android", + context_screen_density: 2, + id: "84e26acc-56a5-4835-8233-591137fca468", + anonymous_id: "00000000000000000000000000", + user_id: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + group_id: "g1", + sent_at: "2019-10-14T09:03:22.563Z", + original_timestamp: "2019-10-14T09:03:17.562Z", + _timestamp: "2020-01-24T11:59:02.403Z", + received_at: "2020-01-24T11:59:02.403Z", + channel: "web" + } + } + ] + } + }, + alias: { + input: { + destination: { + Config: { + apiKey: "dummyApiKey", + prefixProperties: true, + useNativeSDK: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true }, - output: { - default: [ - { - metadata: { - table: "aliases", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - previous_id: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "79313729-7fe5-4204-963a-dc46f4205e4e", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "1234abc", - sent_at: "2020-01-24T06:29:02.366Z", - timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.366Z", - channel: "web", - previous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" - } - } - ], - snowflake: [ - { - metadata: { - table: "ALIASES", - columns: { - UUID_TS: "datetime", - CONTEXT_APP_BUILD: "string", - CONTEXT_APP_NAME: "string", - CONTEXT_APP_NAMESPACE: "string", - CONTEXT_APP_VERSION: "string", - CONTEXT_LIBRARY_NAME: "string", - CONTEXT_LIBRARY_VERSION: "string", - CONTEXT_LOCALE: "string", - CONTEXT_SCREEN_DENSITY: "int", - CONTEXT_TRAITS_CITY: "string", - CONTEXT_TRAITS_COUNTRY: "string", - CONTEXT_TRAITS_EMAIL: "string", - CONTEXT_TRAITS_FIRSTNAME: "string", - CONTEXT_USER_AGENT: "string", - CONTEXT_IP: "string", - CONTEXT_REQUEST_IP: "string", - CONTEXT_PASSED_IP: "string", - ID: "string", - ANONYMOUS_ID: "string", - USER_ID: "string", - SENT_AT: "datetime", - TIMESTAMP: "datetime", - RECEIVED_AT: "datetime", - ORIGINAL_TIMESTAMP: "datetime", - CHANNEL: "string", - PREVIOUS_ID: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - CONTEXT_APP_BUILD: "1.0.0", - CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", - CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", - CONTEXT_APP_VERSION: "1.0.5", - CONTEXT_IP: "0.0.0.0", - CONTEXT_REQUEST_IP: "[::1]:53708", - CONTEXT_PASSED_IP: "0.0.0.0", - CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", - CONTEXT_LIBRARY_VERSION: "1.0.5", - CONTEXT_LOCALE: "en-GB", - CONTEXT_SCREEN_DENSITY: 2, - CONTEXT_TRAITS_CITY: "Disney", - CONTEXT_TRAITS_COUNTRY: "USA", - CONTEXT_TRAITS_EMAIL: "mickey@disney.com", - CONTEXT_TRAITS_FIRSTNAME: "Mickey", - CONTEXT_USER_AGENT: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - ID: "79313729-7fe5-4204-963a-dc46f4205e4e", - ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - USER_ID: "1234abc", - SENT_AT: "2020-01-24T06:29:02.366Z", - TIMESTAMP: "2020-01-24T06:29:02.403Z", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.366Z", - CHANNEL: "web", - PREVIOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" - } - } - ], - s3_datalake: [ - { - metadata: { - table: "aliases", - columns: { - uuid_ts: "datetime", - context_app_build: "string", - context_app_name: "string", - context_app_namespace: "string", - context_app_version: "string", - context_library_name: "string", - context_library_version: "string", - context_locale: "string", - context_screen_density: "int", - context_traits_city: "string", - context_traits_country: "string", - context_traits_email: "string", - context_traits_firstname: "string", - context_user_agent: "string", - context_ip: "string", - context_request_ip: "string", - context_passed_ip: "string", - id: "string", - anonymous_id: "string", - user_id: "string", - sent_at: "datetime", - _timestamp: "datetime", - received_at: "datetime", - original_timestamp: "datetime", - channel: "string", - previous_id: "string" - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - context_app_build: "1.0.0", - context_app_name: "RudderLabs JavaScript SDK", - context_app_namespace: "com.rudderlabs.javascript", - context_app_version: "1.0.5", - context_ip: "0.0.0.0", - context_request_ip: "[::1]:53708", - context_passed_ip: "0.0.0.0", - context_library_name: "RudderLabs JavaScript SDK", - context_library_version: "1.0.5", - context_locale: "en-GB", - context_screen_density: 2, - context_traits_city: "Disney", - context_traits_country: "USA", - context_traits_email: "mickey@disney.com", - context_traits_firstname: "Mickey", - context_user_agent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", - id: "79313729-7fe5-4204-963a-dc46f4205e4e", - anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", - user_id: "1234abc", - sent_at: "2020-01-24T06:29:02.366Z", - _timestamp: "2020-01-24T06:29:02.403Z", - received_at: "2020-01-24T06:29:02.403Z", - original_timestamp: "2020-01-24T06:29:02.366Z", - channel: "web", - previous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" - } - } - ] + DestinationDefinition: { + DisplayName: "Kiss Metrics", + ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", + Name: "KISSMETRICS" + }, + Enabled: true, + ID: "1WhcOCGgj9asZu850HvugU2C3Aq", + Name: "Kiss Metrics", + Transformations: [] + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + os: { + name: "", + version: "" + }, + screen: { + density: 2 + }, + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true + }, + messageId: "79313729-7fe5-4204-963a-dc46f4205e4e", + originalTimestamp: "2020-01-24T06:29:02.366Z", + previousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.366Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "alias", + userId: "1234abc" + }, + request: { + query: { + whSchemaVersion: "v1" } + } }, - extract: { - input: { - destination: { - Config: { - restApiKey: "dummyApiKey", - prefixProperties: true, - useNativeSDK: false, - allowUsersContextTraits: true, - allowEventContextTraits: true, - underscoreDivideNumbers: true - }, - DestinationDefinition: { - DisplayName: "Braze", - ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", - Name: "BRAZE" - }, - Enabled: true, - ID: "1WhcOCGgj9asZu850HvugU2C3Aq", - Name: "Braze", - Transformations: [] + output: { + default: [ + { + metadata: { + table: "aliases", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + previous_id: "string" }, - message: { - channel: "sources", - context: { - sources: { - job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", - job_run_id: "cfd6705nsevh5p2l77ag", - task_run_id: "cfd6705nsevh5p2l77b0", - version: "version" - } - }, - properties: { - string_property: "value1", - date_property: "2023-02-01T07:53:31.430Z", - boolean_property: true, - date_property2: "2023-01-31T07:39:10.002Z", - userId: "dummy-user-id-inside-properties" - }, - event: "extract_event", - recordId: "some-uuid", - originalTimestamp: "2020-01-24T06:29:02.364Z", - receivedAt: "2020-01-24T11:59:02.403+05:30", - request_ip: "[::1]:53708", - sentAt: "2020-01-24T06:29:02.364Z", - timestamp: "2020-01-24T11:59:02.403+05:30", - type: "extract", - userId: "dummy-user-id-outside-properties" + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "79313729-7fe5-4204-963a-dc46f4205e4e", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "1234abc", + sent_at: "2020-01-24T06:29:02.366Z", + timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.366Z", + channel: "web", + previous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" + } + } + ], + snowflake: [ + { + metadata: { + table: "ALIASES", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + PREVIOUS_ID: "string" }, - request: { - query: { - whSchemaVersion: "v1" - } + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "79313729-7fe5-4204-963a-dc46f4205e4e", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "1234abc", + SENT_AT: "2020-01-24T06:29:02.366Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.366Z", + CHANNEL: "web", + PREVIOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" + } + } + ], + s3_datalake: [ + { + metadata: { + table: "aliases", + columns: { + uuid_ts: "datetime", + context_app_build: "string", + context_app_name: "string", + context_app_namespace: "string", + context_app_version: "string", + context_library_name: "string", + context_library_version: "string", + context_locale: "string", + context_screen_density: "int", + context_traits_city: "string", + context_traits_country: "string", + context_traits_email: "string", + context_traits_firstname: "string", + context_user_agent: "string", + context_ip: "string", + context_request_ip: "string", + context_passed_ip: "string", + id: "string", + anonymous_id: "string", + user_id: "string", + sent_at: "datetime", + _timestamp: "datetime", + received_at: "datetime", + original_timestamp: "datetime", + channel: "string", + previous_id: "string" }, - metadata: { - sourceCategory: "singer-protocol", - } + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + context_app_build: "1.0.0", + context_app_name: "RudderLabs JavaScript SDK", + context_app_namespace: "com.rudderlabs.javascript", + context_app_version: "1.0.5", + context_ip: "0.0.0.0", + context_request_ip: "[::1]:53708", + context_passed_ip: "0.0.0.0", + context_library_name: "RudderLabs JavaScript SDK", + context_library_version: "1.0.5", + context_locale: "en-GB", + context_screen_density: 2, + context_traits_city: "Disney", + context_traits_country: "USA", + context_traits_email: "mickey@disney.com", + context_traits_firstname: "Mickey", + context_user_agent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + id: "79313729-7fe5-4204-963a-dc46f4205e4e", + anonymous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + user_id: "1234abc", + sent_at: "2020-01-24T06:29:02.366Z", + _timestamp: "2020-01-24T06:29:02.403Z", + received_at: "2020-01-24T06:29:02.403Z", + original_timestamp: "2020-01-24T06:29:02.366Z", + channel: "web", + previous_id: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" + } + } + ] + } + }, + extract: { + input: { + destination: { + Config: { + restApiKey: "dummyApiKey", + prefixProperties: true, + useNativeSDK: false, + allowUsersContextTraits: true, + allowEventContextTraits: true, + underscoreDivideNumbers: true }, - output: { - default: [ - { - metadata: { - table: "extract_event", - columns: { - date_property: "datetime", - date_property_2: "datetime", - boolean_property: "boolean", - context_sources_job_id: "string", - context_sources_job_run_id: "string", - context_sources_task_run_id: "string", - context_sources_version: "string", - event: "string", - id: "string", - user_id: "string", - received_at: "datetime", - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - date_property: "2023-02-01T07:53:31.430Z", - date_property_2: "2023-01-31T07:39:10.002Z", - context_sources_job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", - context_sources_job_run_id: "cfd6705nsevh5p2l77ag", - context_sources_task_run_id: "cfd6705nsevh5p2l77b0", - context_sources_version: "version", - id: "some-uuid", - user_id: "dummy-user-id-inside-properties", - received_at: "2020-01-24T06:29:02.403Z", - event: "extract_event", - boolean_property: true - } - } - ], - snowflake: [ - { - metadata: { - table: "EXTRACT_EVENT", - columns: { - DATE_PROPERTY: "datetime", - DATE_PROPERTY_2: "datetime", - CONTEXT_SOURCES_JOB_ID: "string", - CONTEXT_SOURCES_JOB_RUN_ID: "string", - CONTEXT_SOURCES_TASK_RUN_ID: "string", - CONTEXT_SOURCES_VERSION: "string", - EVENT: "string", - ID: "string", - USER_ID: "string", - RECEIVED_AT: "datetime", - BOOLEAN_PROPERTY: "boolean", - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - DATE_PROPERTY: "2023-02-01T07:53:31.430Z", - DATE_PROPERTY_2: "2023-01-31T07:39:10.002Z", - CONTEXT_SOURCES_JOB_ID: "2JABSy1nq89H7xeJimBL2pCtOOp", - CONTEXT_SOURCES_JOB_RUN_ID: "cfd6705nsevh5p2l77ag", - CONTEXT_SOURCES_TASK_RUN_ID: "cfd6705nsevh5p2l77b0", - CONTEXT_SOURCES_VERSION: "version", - ID: "some-uuid", - USER_ID: "dummy-user-id-inside-properties", - RECEIVED_AT: "2020-01-24T06:29:02.403Z", - EVENT: "extract_event", - BOOLEAN_PROPERTY: true, - } - } - ], - s3_datalake: [ - { - metadata: { - table: "extract_event", - columns: { - date_property: "datetime", - date_property_2: "datetime", - context_sources_job_id: "string", - context_sources_job_run_id: "string", - context_sources_task_run_id: "string", - context_sources_version: "string", - event: "string", - id: "string", - user_id: "string", - received_at: "datetime", - boolean_property: "boolean", - }, - receivedAt: "2020-01-24T11:59:02.403+05:30" - }, - data: { - date_property: "2023-02-01T07:53:31.430Z", - date_property_2: "2023-01-31T07:39:10.002Z", - context_sources_job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", - context_sources_job_run_id: "cfd6705nsevh5p2l77ag", - context_sources_task_run_id: "cfd6705nsevh5p2l77b0", - context_sources_version: "version", - id: "some-uuid", - event: "extract_event", - boolean_property: true, - received_at: "2020-01-24T06:29:02.403Z", - user_id: "dummy-user-id-inside-properties", - } - } - ] + DestinationDefinition: { + DisplayName: "Braze", + ID: "1WhbSZ6uA3H5ChVifHpfL2H6sie", + Name: "BRAZE" + }, + Enabled: true, + ID: "1WhcOCGgj9asZu850HvugU2C3Aq", + Name: "Braze", + Transformations: [] + }, + message: { + channel: "sources", + context: { + sources: { + job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", + job_run_id: "cfd6705nsevh5p2l77ag", + task_run_id: "cfd6705nsevh5p2l77b0", + version: "version" + } + }, + properties: { + string_property: "value1", + date_property: "2023-02-01T07:53:31.430Z", + boolean_property: true, + date_property2: "2023-01-31T07:39:10.002Z", + userId: "dummy-user-id-inside-properties" + }, + event: "extract_event", + recordId: "some-uuid", + originalTimestamp: "2020-01-24T06:29:02.364Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "extract", + userId: "dummy-user-id-outside-properties" + }, + request: { + query: { + whSchemaVersion: "v1" } + }, + metadata: { + sourceCategory: "singer-protocol", + } }, + output: { + default: [ + { + metadata: { + table: "extract_event", + columns: { + date_property: "datetime", + date_property_2: "datetime", + boolean_property: "boolean", + context_sources_job_id: "string", + context_sources_job_run_id: "string", + context_sources_task_run_id: "string", + context_sources_version: "string", + event: "string", + id: "string", + user_id: "string", + received_at: "datetime", + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + date_property: "2023-02-01T07:53:31.430Z", + date_property_2: "2023-01-31T07:39:10.002Z", + context_sources_job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", + context_sources_job_run_id: "cfd6705nsevh5p2l77ag", + context_sources_task_run_id: "cfd6705nsevh5p2l77b0", + context_sources_version: "version", + id: "some-uuid", + user_id: "dummy-user-id-inside-properties", + received_at: "2020-01-24T06:29:02.403Z", + event: "extract_event", + boolean_property: true + } + } + ], + snowflake: [ + { + metadata: { + table: "EXTRACT_EVENT", + columns: { + DATE_PROPERTY: "datetime", + DATE_PROPERTY_2: "datetime", + CONTEXT_SOURCES_JOB_ID: "string", + CONTEXT_SOURCES_JOB_RUN_ID: "string", + CONTEXT_SOURCES_TASK_RUN_ID: "string", + CONTEXT_SOURCES_VERSION: "string", + EVENT: "string", + ID: "string", + USER_ID: "string", + RECEIVED_AT: "datetime", + BOOLEAN_PROPERTY: "boolean", + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + DATE_PROPERTY: "2023-02-01T07:53:31.430Z", + DATE_PROPERTY_2: "2023-01-31T07:39:10.002Z", + CONTEXT_SOURCES_JOB_ID: "2JABSy1nq89H7xeJimBL2pCtOOp", + CONTEXT_SOURCES_JOB_RUN_ID: "cfd6705nsevh5p2l77ag", + CONTEXT_SOURCES_TASK_RUN_ID: "cfd6705nsevh5p2l77b0", + CONTEXT_SOURCES_VERSION: "version", + ID: "some-uuid", + USER_ID: "dummy-user-id-inside-properties", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + EVENT: "extract_event", + BOOLEAN_PROPERTY: true, + } + } + ], + s3_datalake: [ + { + metadata: { + table: "extract_event", + columns: { + date_property: "datetime", + date_property_2: "datetime", + context_sources_job_id: "string", + context_sources_job_run_id: "string", + context_sources_task_run_id: "string", + context_sources_version: "string", + event: "string", + id: "string", + user_id: "string", + received_at: "datetime", + boolean_property: "boolean", + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + date_property: "2023-02-01T07:53:31.430Z", + date_property_2: "2023-01-31T07:39:10.002Z", + context_sources_job_id: "2JABSy1nq89H7xeJimBL2pCtOOp", + context_sources_job_run_id: "cfd6705nsevh5p2l77ag", + context_sources_task_run_id: "cfd6705nsevh5p2l77b0", + context_sources_version: "version", + id: "some-uuid", + event: "extract_event", + boolean_property: true, + received_at: "2020-01-24T06:29:02.403Z", + user_id: "dummy-user-id-inside-properties", + } + } + ] + } + }, }; function input(eventType) { - return _.cloneDeep(sampleEvents[eventType].input); + return _.cloneDeep(sampleEvents[eventType].input); } function output(eventType, provider) { - if (provider === "snowflake") { - return _.cloneDeep(sampleEvents[eventType].output.snowflake); - } - if (provider === "s3_datalake") { - return _.cloneDeep(sampleEvents[eventType].output.s3_datalake); - } - return _.cloneDeep(sampleEvents[eventType].output.default); + if (provider === "snowflake") { + return _.cloneDeep(sampleEvents[eventType].output.snowflake); + } + if (provider === "s3_datalake") { + return _.cloneDeep(sampleEvents[eventType].output.s3_datalake); + } + return _.cloneDeep(sampleEvents[eventType].output.default); } -module.exports = {input, output}; +module.exports = { input, output }; diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index d1fc833fe0..84371e40b3 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -48,1040 +48,1156 @@ const integrationCasedString = (integration, str) => { }; describe("event types", () => { - describe("track", () => { - it("should generate two events for every track call", () => { - const i = input("track"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("track", integrations[index])); - }); - }); + describe("track", () => { + it("should generate two events for every track call", () => { + const i = input("track"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("track", integrations[index])); + }); }); - - describe("identify", () => { - it("should generate two events for every identify call", () => { - const i = input("identify"); - // also verfies priority order between traits and context.traits - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("identify", integrations[index])); - }); - }); + }); + + describe("identify", () => { + it("should generate two events for every identify call", () => { + const i = input("identify"); + // also verfies priority order between traits and context.traits + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("identify", integrations[index])); + }); }); - - describe("page", () => { - it("should generate one event for every page call", () => { - const i = input("page"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("page", integrations[index])); - }); - }); - it("should take name from properties if top-level name is missing", () => { - let i = input("page"); - i.message.properties.name = i.message.name; - delete i.message.name; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("page", integrations[index])); - }); - }); + }); + + describe("page", () => { + it("should generate one event for every page call", () => { + const i = input("page"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("page", integrations[index])); + }); }); - - describe("screen", () => { - it("should generate one event for every screen call", () => { - const i = input("screen"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integrations[index])); - }); - }); - it("should take name from properties if top-level name is missing", () => { - let i = input("screen"); - i.message.properties.name = i.message.name; - delete i.message.name; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integrations[index])); - }); - }); + it("should take name from properties if top-level name is missing", () => { + let i = input("page"); + i.message.properties.name = i.message.name; + delete i.message.name; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("page", integrations[index])); + }); }); - - describe("alias", () => { - it("should generate one event for every alias call", () => { - const i = input("alias"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("alias", integrations[index])); - }); - }); + }); + + describe("screen", () => { + it("should generate one event for every screen call", () => { + const i = input("screen"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("screen", integrations[index])); + }); }); - - describe("extract", () => { - it("should generate one event for every extract call", () => { - const i = input("extract"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toMatchObject(output("extract", integrations[index])); - }); - }); + it("should take name from properties if top-level name is missing", () => { + let i = input("screen"); + i.message.properties.name = i.message.name; + delete i.message.name; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("screen", integrations[index])); + }); + }); + }); + + describe("alias", () => { + it("should generate one event for every alias call", () => { + const i = input("alias"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("alias", integrations[index])); + }); }); + }); + + describe("extract", () => { + it("should generate one event for every extract call", () => { + const i = input("extract"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toMatchObject(output("extract", integrations[index])); + }); + }); + }); }); describe("column & table names", () => { - it("should handle special character, spacing, snake case etc.", () => { - let i = input("track"); - - i.message.properties = Object.assign( - i.message.properties, - names.input.properties + it("should handle special character, spacing, snake case etc.", () => { + let i = input("track"); + + i.message.properties = Object.assign( + i.message.properties, + names.input.properties + ); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + + const provider = + integrations[index] === "snowflake" ? "snowflake" : "default"; + + expect(received[1].metadata.columns).toMatchObject( + names.output.columns[provider] + ); + expect(received[1].data).toMatchObject(names.output.data[provider]); + + for (let tableName in names.input.properties) { + i.message.event = tableName; + let out = transformer.process(i); + expect(out[1].metadata.table).toEqual( + names.output.namesMap[provider][tableName] ); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - - const provider = - integrations[index] === "snowflake" ? "snowflake" : "default"; - - expect(received[1].metadata.columns).toMatchObject( - names.output.columns[provider] - ); - expect(received[1].data).toMatchObject(names.output.data[provider]); - - for (let tableName in names.input.properties) { - i.message.event = tableName; - let out = transformer.process(i); - expect(out[1].metadata.table).toEqual( - names.output.namesMap[provider][tableName] - ); - } - }); + } }); - - it("should trim column names in postgres", () => { - let i = input("track"); - names.input.properties[ - "a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5" - ] = "70 letter identifier"; - names.input.properties[ - "xa1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5abcdefghi" - ] = "500 letter identifier"; - i.message.properties = Object.assign( - i.message.properties, - names.input.properties + }); + it("should trim column names in postgres", () => { + let i = input("track"); + names.input.properties[ + "a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5" + ] = "70 letter identifier"; + names.input.properties[ + "xa1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5abcdefghi" + ] = "500 letter identifier"; + i.message.properties = Object.assign( + i.message.properties, + names.input.properties + ); + i.message.event = + "a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5"; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + if (integrations[index] === "postgres") { + expect(received[1].metadata).toHaveProperty( + "table", + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1" ); - i.message.event = - "a1a2a3a4a5b1b2b3b4b5c1c2c3c4c5d1d2d3d4d5e1e2e3e4e5f1f2f3f4f5g1g2g3g4g5"; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - if (integrations[index] === "postgres") { - expect(received[1].metadata).toHaveProperty( - "table", - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1" - ); - expect(received[1].metadata.columns).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1", - "string" - ); - expect(received[1].data).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1", - "70 letter identifier" - ); - expect(received[1].metadata.columns).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_", - "string" - ); - expect(received[1].data).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_", - "500 letter identifier" - ); - //KEY should be trimmed to 63 - return; - } - if (integrations[index] === "snowflake") { - expect(received[1].metadata).toHaveProperty( - "table", - "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2" - ); - expect(received[1].metadata.columns).toHaveProperty( - "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2", - "string" - ); - expect(received[1].data).toHaveProperty( - "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2", - "70 letter identifier" - ); - expect(received[1].metadata.columns).toHaveProperty( - "XA_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_", - "string" - ); - expect(received[1].data).toHaveProperty( - "XA_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_", - "500 letter identifier" - ); - return; - } - if (integrations[index] === "s3_datalake" || integrations[index] === "gcs_datalake" || integrations[index] === "azure_datalake") { - expect(received[1].metadata).toHaveProperty( - "table", - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5" - ); - expect(received[1].metadata.columns).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5", - "string" - ); - expect(received[1].data).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5", - "70 letter identifier" - ); - expect(received[1].metadata.columns).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_abcdefghi", - "string" - ); - expect(received[1].data).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_abcdefghi", - "500 letter identifier" - ); - return; - } - expect(received[1].metadata).toHaveProperty( - "table", - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2" - ); - expect(received[1].metadata.columns).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2", - "string" - ); - expect(received[1].data).toHaveProperty( - "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2", - "70 letter identifier" - ); - expect(received[1].metadata.columns).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_", - "string" - ); - expect(received[1].data).toHaveProperty( - "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_", - "500 letter identifier" - ); - //KEY should be trimmed to 127 - }); + expect(received[1].metadata.columns).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1", + "string" + ); + expect(received[1].data).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1", + "70 letter identifier" + ); + expect(received[1].metadata.columns).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_", + "string" + ); + expect(received[1].data).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_", + "500 letter identifier" + ); + //KEY should be trimmed to 63 + return; + } + if (integrations[index] === "snowflake") { + expect(received[1].metadata).toHaveProperty( + "table", + "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2" + ); + expect(received[1].metadata.columns).toHaveProperty( + "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2", + "string" + ); + expect(received[1].data).toHaveProperty( + "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2", + "70 letter identifier" + ); + expect(received[1].metadata.columns).toHaveProperty( + "XA_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_", + "string" + ); + expect(received[1].data).toHaveProperty( + "XA_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_", + "500 letter identifier" + ); + return; + } + if (integrations[index] === "s3_datalake" || integrations[index] === "gcs_datalake" || integrations[index] === "azure_datalake") { + expect(received[1].metadata).toHaveProperty( + "table", + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5" + ); + expect(received[1].metadata.columns).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5", + "string" + ); + expect(received[1].data).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5", + "70 letter identifier" + ); + expect(received[1].metadata.columns).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_abcdefghi", + "string" + ); + expect(received[1].data).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5_abcdefghi", + "500 letter identifier" + ); + return; + } + expect(received[1].metadata).toHaveProperty( + "table", + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2" + ); + expect(received[1].metadata.columns).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2", + "string" + ); + expect(received[1].data).toHaveProperty( + "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2", + "70 letter identifier" + ); + expect(received[1].metadata.columns).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_", + "string" + ); + expect(received[1].data).toHaveProperty( + "xa_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_", + "500 letter identifier" + ); + //KEY should be trimmed to 127 }); + }); }); // tests case where properties set by user match the columns set by rudder(id, recevied etc) // https://docs.rudderstack.com/data-warehouse-integrations/warehouse-schemas#standard-rudderstack-properties describe("conflict between rudder set props and user set props", () => { - it("should override user set props with rudder prop", () => { - eventTypes.forEach(evType => { - const i = input(evType); - - const propsKey = propsKeyMap[evType]; - const rudderProps = _.compact( - rudderProperties.default.concat(rudderProperties[evType]) - ); - - rudderProps.forEach(prop => { - // _.set creates inner object if not present - _.set(i.message, `${propsKey}.${prop}`, "test prop"); - }); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - rudderProps.forEach(prop => { - if (received[0].data[prop] === "test prop") { - } - expect(received[0].data[prop]).not.toBe("test prop"); - }); - }); + it("should override user set props with rudder prop", () => { + eventTypes.forEach(evType => { + const i = input(evType); + + const propsKey = propsKeyMap[evType]; + const rudderProps = _.compact( + rudderProperties.default.concat(rudderProperties[evType]) + ); + + rudderProps.forEach(prop => { + // _.set creates inner object if not present + _.set(i.message, `${propsKey}.${prop}`, "test prop"); + }); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + rudderProps.forEach(prop => { + if (received[0].data[prop] === "test prop") { + } + expect(received[0].data[prop]).not.toBe("test prop"); }); + }); }); + }); - it("should set data type of rudder prop and not user set prop", () => { - eventTypes.forEach(evType => { - let i = input(evType); + it("should set data type of rudder prop and not user set prop", () => { + eventTypes.forEach(evType => { + let i = input(evType); - const propsKey = propsKeyMap[evType]; - transformers.forEach((transformer, index) => { - let sampleRudderPropKey = "id"; - if (integrations[index] === "snowflake") { - sampleRudderPropKey = "ID"; - } + const propsKey = propsKeyMap[evType]; + transformers.forEach((transformer, index) => { + let sampleRudderPropKey = "id"; + if (integrations[index] === "snowflake") { + sampleRudderPropKey = "ID"; + } - _.set(i.message, `${propsKey}.${sampleRudderPropKey}`, true); - const received = transformer.process(i); - expect(received[0].metadata.columns[sampleRudderPropKey]).not.toBe( - "bool" - ); - expect(received[0].metadata.columns[sampleRudderPropKey]).toBe( - "string" - ); - }); - }); + _.set(i.message, `${propsKey}.${sampleRudderPropKey}`, true); + const received = transformer.process(i); + expect(received[0].metadata.columns[sampleRudderPropKey]).not.toBe( + "bool" + ); + expect(received[0].metadata.columns[sampleRudderPropKey]).toBe( + "string" + ); + }); }); + }); }); describe("handle reserved words", () => { - const OLD_ENV = process.env; - beforeEach(() => { - process.env = {...OLD_ENV}; - process.env.WH_MAX_COLUMNS_IN_EVENT = 600; - jest.resetModules(); - }); - afterAll(() => { - process.env = OLD_ENV; // restore old env - }); - it("prepend underscore", () => { - // re-import transformer modules so that new env values are used - const transformers = integrations.map(integration => - require(`../../src/${version}/destinations/${integration}/transform`) + const OLD_ENV = process.env; + beforeEach(() => { + process.env = { ...OLD_ENV }; + process.env.WH_MAX_COLUMNS_IN_EVENT = 600; + jest.resetModules(); + }); + afterAll(() => { + process.env = OLD_ENV; // restore old env + }); + it("prepend underscore", () => { + // re-import transformer modules so that new env values are used + const transformers = integrations.map(integration => + require(`../../src/${version}/destinations/${integration}/transform`) + ); + eventTypes.forEach(evType => { + let i = input(evType); + + const propsKey = propsKeyMap[evType]; + transformers.forEach((transformer, index) => { + const reserverdKeywordsMap = + reservedANSIKeywordsMap[integrations[index].toUpperCase()]; + + i.message[propsKey] = Object.assign( + i.message[propsKey] || {}, + reserverdKeywordsMap ); - eventTypes.forEach(evType => { - let i = input(evType); - - const propsKey = propsKeyMap[evType]; - transformers.forEach((transformer, index) => { - const reserverdKeywordsMap = - reservedANSIKeywordsMap[integrations[index].toUpperCase()]; - - i.message[propsKey] = Object.assign( - i.message[propsKey] || {}, - reserverdKeywordsMap - ); - - const received = transformer.process(i); - - const out = - evType === "track" || evType === "identify" - ? received[1] - : received[0]; - - Object.keys(reserverdKeywordsMap).forEach(k => { - expect(out.metadata.columns).not.toHaveProperty(k.toLowerCase()); - expect(out.metadata.columns).not.toHaveProperty(k.toUpperCase()); - let snakeCasedKey = _.snakeCase(k).toUpperCase(); - if (k === snakeCasedKey) { - k = `_${k}`; - } else { - k = snakeCasedKey; - } - if (integrations[index] === "snowflake") { - expect(out.metadata.columns).toHaveProperty(k); - } else { - expect(out.metadata.columns).toHaveProperty(k.toLowerCase()); - } - }); - }); + + const received = transformer.process(i); + + const out = + evType === "track" || evType === "identify" + ? received[1] + : received[0]; + + Object.keys(reserverdKeywordsMap).forEach(k => { + expect(out.metadata.columns).not.toHaveProperty(k.toLowerCase()); + expect(out.metadata.columns).not.toHaveProperty(k.toUpperCase()); + let snakeCasedKey = _.snakeCase(k).toUpperCase(); + if (k === snakeCasedKey) { + k = `_${k}`; + } else { + k = snakeCasedKey; + } + if (integrations[index] === "snowflake") { + expect(out.metadata.columns).toHaveProperty(k); + } else { + expect(out.metadata.columns).toHaveProperty(k.toLowerCase()); + } }); + }); }); + }); }); describe("null/empty values", () => { - it("should skip setting null/empty value fields", () => { - eventTypes.forEach(evType => { - let i = input(evType); - - const propsKey = propsKeyMap[evType]; - const emptyValsMap = { - empty_val_1: "", - empty_val_2: [], - empty_val_3: null, - empty_val_4: undefined - }; - const emptyValKeys = Object.keys(emptyValsMap).concat( - Object.keys(emptyValsMap).map(k => k.toUpperCase()) - ); - - i.message[propsKey] = Object.assign( - i.message[propsKey] || {}, - emptyValsMap - ); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - emptyValKeys.forEach(k => { - expect(received[0].metadata.columns).not.toHaveProperty(k); - expect(received[0].data).not.toHaveProperty(k); - if (received[1]) { - expect(received[1].metadata.columns).not.toHaveProperty(k); - expect(received[1].data).not.toHaveProperty(k); - } - }); - }); + it("should skip setting null/empty value fields", () => { + eventTypes.forEach(evType => { + let i = input(evType); + + const propsKey = propsKeyMap[evType]; + const emptyValsMap = { + empty_val_1: "", + empty_val_2: [], + empty_val_3: null, + empty_val_4: undefined + }; + const emptyValKeys = Object.keys(emptyValsMap).concat( + Object.keys(emptyValsMap).map(k => k.toUpperCase()) + ); + + i.message[propsKey] = Object.assign( + i.message[propsKey] || {}, + emptyValsMap + ); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + emptyValKeys.forEach(k => { + expect(received[0].metadata.columns).not.toHaveProperty(k); + expect(received[0].data).not.toHaveProperty(k); + if (received[1]) { + expect(received[1].metadata.columns).not.toHaveProperty(k); + expect(received[1].data).not.toHaveProperty(k); + } }); + }); }); + }); }); describe("invalid context", () => { - it("should skip setting context fields if context is not an object", () => { - eventTypes.forEach(evType => { - let i = input(evType); - i.message.context = "{{invalid object}}"; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - let columns = Object.keys(received[0].metadata.columns); - if (received[1]) { - columns = columns.concat(Object.keys(received[1].metadata.columns)); - } - columns.forEach(c => { - expect(c).not.toMatch(/^context_\d/g); - }); - let data = Object.keys(received[0].data); - if (received[1]) { - data = data.concat(Object.keys(received[1].data)); - } - data.forEach(d => { - expect(d).not.toMatch(/^context_\d/g); - }); - }); + it("should skip setting context fields if context is not an object", () => { + eventTypes.forEach(evType => { + let i = input(evType); + i.message.context = "{{invalid object}}"; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + let columns = Object.keys(received[0].metadata.columns); + if (received[1]) { + columns = columns.concat(Object.keys(received[1].metadata.columns)); + } + columns.forEach(c => { + expect(c).not.toMatch(/^context_\d/g); }); + let data = Object.keys(received[0].data); + if (received[1]) { + data = data.concat(Object.keys(received[1].data)); + } + data.forEach(d => { + expect(d).not.toMatch(/^context_\d/g); + }); + }); }); + }); }); describe("context ip", () => { - it("should set context_ip to context.ip if present", () => { - eventTypes.forEach(evType => { - let i = input(evType); - i.message.context.ip = "new_ip"; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toBe("string"); - expect( - received[0].data[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toEqual("new_ip"); - - if (received[1]) { - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toBe("string"); - expect( - received[1].data[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toEqual("new_ip"); - } - }); - }); + it("should set context_ip to context.ip if present", () => { + eventTypes.forEach(evType => { + let i = input(evType); + i.message.context.ip = "new_ip"; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toBe("string"); + expect( + received[0].data[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toEqual("new_ip"); + + if (received[1]) { + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toBe("string"); + expect( + received[1].data[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toEqual("new_ip"); + } + }); }); - - it("should set context_ip to request_ip if context.ip not present", () => { - eventTypes.forEach(evType => { - let i = input(evType); - delete i.message.context.ip; - i.message.request_ip = "requested_ip"; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toBe("string"); - expect( - received[0].data[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toEqual("requested_ip"); - if (received[1]) { - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toBe("string"); - expect( - received[1].data[ - integrationCasedString(integrations[index], "context_ip") - ] - ).toEqual("requested_ip"); - } - }); - }); + }); + + it("should set context_ip to request_ip if context.ip not present", () => { + eventTypes.forEach(evType => { + let i = input(evType); + delete i.message.context.ip; + i.message.request_ip = "requested_ip"; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toBe("string"); + expect( + received[0].data[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toEqual("requested_ip"); + if (received[1]) { + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toBe("string"); + expect( + received[1].data[ + integrationCasedString(integrations[index], "context_ip") + ] + ).toEqual("requested_ip"); + } + }); }); + }); }); describe("remove rudder property if rudder property is null", () => { - it("should remove context_ip set by user in properties if missing in event", () => { - eventTypes.forEach(evType => { - let i = input(evType); - delete i.message.context.ip; - delete i.message.request_ip; - const propsKey = propsKeyMap[evType]; - _.set(i.message, `${propsKey}.context_ip`, "test prop"); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "context_ip") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "context_ip") - ); - }); - }); + it("should remove context_ip set by user in properties if missing in event", () => { + eventTypes.forEach(evType => { + let i = input(evType); + delete i.message.context.ip; + delete i.message.request_ip; + const propsKey = propsKeyMap[evType]; + _.set(i.message, `${propsKey}.context_ip`, "test prop"); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "context_ip") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "context_ip") + ); + }); }); + }); }); - describe("remove any property if event is object ", () => { - it("should remove any property if event is object", () => { - eventTypes.forEach(evType => { - let i = input(evType); - i.message.channel = {}; - i.message.event = {}; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "channel") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "channel") - ); - }); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "event_text") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "event_text") - ); - }); - i.message.channel = {channel: "android"}; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "channel") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "channel") - ); - }); - }); + it("should remove any property if event is object", () => { + eventTypes.forEach(evType => { + let i = input(evType); + i.message.channel = {}; + i.message.event = {}; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "channel") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "channel") + ); + }); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "event_text") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "event_text") + ); + }); + i.message.channel = { channel: "android" }; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "channel") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "channel") + ); + }); }); + }); }); describe("store full rudder event", () => { - it("should store if configured in dest settings", () => { - eventTypes.forEach(evType => { - let i = input(evType); - _.set(i.destination, `Config.storeFullEvent`, true); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - const columnName = integrationCasedString( - integrations[index], - "rudder_event" - ); - - expect(received[0].metadata.columns).toHaveProperty(columnName); - expect(received[0].metadata.columns[columnName]).toEqual( - fullEventColumnTypeByProvider[integrations[index]] - ); - expect(received[0].data[columnName]).toEqual(JSON.stringify(i.message)); - - if (received[1]) { - expect(received[1].metadata.columns).not.toHaveProperty(columnName); - expect(received[1].data).not.toHaveProperty(columnName); - } - }); - }); + it("should store if configured in dest settings", () => { + eventTypes.forEach(evType => { + let i = input(evType); + _.set(i.destination, `Config.storeFullEvent`, true); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + const columnName = integrationCasedString( + integrations[index], + "rudder_event" + ); + + expect(received[0].metadata.columns).toHaveProperty(columnName); + expect(received[0].metadata.columns[columnName]).toEqual( + fullEventColumnTypeByProvider[integrations[index]] + ); + expect(received[0].data[columnName]).toEqual(JSON.stringify(i.message)); + + if (received[1]) { + expect(received[1].metadata.columns).not.toHaveProperty(columnName); + expect(received[1].data).not.toHaveProperty(columnName); + } + }); }); + }); }); describe("rudder reserved columns", () => { - it("should not accept rudder reserved column names from user in properties, traits etc", () => { - eventTypes.forEach(evType => { - let i = input(evType); - - const delProps = [ - "message.channel", - "message.timestamp", - "message.originalTimestamp" - ]; - const setProps = [ - "message.properties.channel", - "message.traits.channel", - "message.properties.timestamp", - "message.traits.timestamp", - "message.properties.originalTimestamp", - "message.traits.originalTimestamp" - ]; - - const checkProps = ["channel", "timestamp", "original_timestamp"]; - - delProps.forEach(prop => _.unset(i, prop)); - setProps.forEach(prop => _.set(i, prop, "random value")); - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - checkProps.forEach(k => { - k = integrationCasedString(integrations[index], k); - expect(received[0].metadata.columns).not.toHaveProperty(k); - expect(received[0].data).not.toHaveProperty(k); - if (received[1]) { - expect(received[1].metadata.columns).not.toHaveProperty(k); - expect(received[1].data).not.toHaveProperty(k); - } - }); - }); + it("should not accept rudder reserved column names from user in properties, traits etc", () => { + eventTypes.forEach(evType => { + let i = input(evType); + + const delProps = [ + "message.channel", + "message.timestamp", + "message.originalTimestamp" + ]; + const setProps = [ + "message.properties.channel", + "message.traits.channel", + "message.properties.timestamp", + "message.traits.timestamp", + "message.properties.originalTimestamp", + "message.traits.originalTimestamp" + ]; + + const checkProps = ["channel", "timestamp", "original_timestamp"]; + + delProps.forEach(prop => _.unset(i, prop)); + setProps.forEach(prop => _.set(i, prop, "random value")); + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + checkProps.forEach(k => { + k = integrationCasedString(integrations[index], k); + expect(received[0].metadata.columns).not.toHaveProperty(k); + expect(received[0].data).not.toHaveProperty(k); + if (received[1]) { + expect(received[1].metadata.columns).not.toHaveProperty(k); + expect(received[1].data).not.toHaveProperty(k); + } }); + }); }); + }); }); describe("id column datatype for users table", () => { - it("should set id column datatype for users as one received and not always string", () => { - let i = input("identify"); - i.message.userId = 100; //integer - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "user_id") - ] - ).toEqual("int"); - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "id") - ] - ).toEqual("int"); - }); + it("should set id column datatype for users as one received and not always string", () => { + let i = input("identify"); + i.message.userId = 100; //integer + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "user_id") + ] + ).toEqual("int"); + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "id") + ] + ).toEqual("int"); + }); - i = input("identify"); - i.message.userId = 1.1; //float - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "user_id") - ] - ).toEqual("float"); - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "id") - ] - ).toEqual("float"); - }); + i = input("identify"); + i.message.userId = 1.1; //float + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "user_id") + ] + ).toEqual("float"); + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "id") + ] + ).toEqual("float"); }); + }); }); describe("handle leading underscores in properties", () => { - it("should not remove leading underscores in properties", () => { - let i = input("track"); - i.message.properties = { - _timestamp: "1", - __timestamp: "2", - __timestamp_new: "3" - }; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "_timestamp") - ); - expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "__timestamp") - ); - expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "__timestamp_new") - ); - expect(received[1].data).toHaveProperty( - integrationCasedString(integrations[index], "_timestamp") - ); - expect(received[1].data).toHaveProperty( - integrationCasedString(integrations[index], "__timestamp") - ); - expect(received[1].data).toHaveProperty( - integrationCasedString(integrations[index], "__timestamp_new") - ); - }); + it("should not remove leading underscores in properties", () => { + let i = input("track"); + i.message.properties = { + _timestamp: "1", + __timestamp: "2", + __timestamp_new: "3" + }; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[1].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "_timestamp") + ); + expect(received[1].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "__timestamp") + ); + expect(received[1].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "__timestamp_new") + ); + expect(received[1].data).toHaveProperty( + integrationCasedString(integrations[index], "_timestamp") + ); + expect(received[1].data).toHaveProperty( + integrationCasedString(integrations[index], "__timestamp") + ); + expect(received[1].data).toHaveProperty( + integrationCasedString(integrations[index], "__timestamp_new") + ); }); + }); }); describe("handle recordId from cloud sources", () => { - it("should not set id based on recordId if sourceCategory is missing", () => { - let i = input("track"); - i.message.recordId = 42; - if (i.metadata) delete i.metadata.sourceCategory; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect( - received[0].data[integrationCasedString(integrations[index], "id")] - ).toEqual(i.message.messageId); - expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[1].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect( - received[1].data[integrationCasedString(integrations[index], "id")] - ).toEqual(i.message.messageId); - }); + it("should not set id based on recordId if sourceCategory is missing", () => { + let i = input("track"); + i.message.recordId = 42; + if (i.metadata) delete i.metadata.sourceCategory; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect( + received[0].data[integrationCasedString(integrations[index], "id")] + ).toEqual(i.message.messageId); + expect(received[1].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[1].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect( + received[1].data[integrationCasedString(integrations[index], "id")] + ).toEqual(i.message.messageId); }); - - it("should not set id based on recordId if source version is missing", () => { - let i = input("track"); - i.message.recordId = 42; - i.metadata = {sourceCategory: "cloud"}; - if (i.message.context.sources) delete i.message.context.sources.version; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect( - received[0].data[integrationCasedString(integrations[index], "id")] - ).toEqual(i.message.messageId); - expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[1].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect( - received[1].data[integrationCasedString(integrations[index], "id")] - ).toEqual(i.message.messageId); - }); + }); + + it("should not set id based on recordId if source version is missing", () => { + let i = input("track"); + i.message.recordId = 42; + i.metadata = { sourceCategory: "cloud" }; + if (i.message.context.sources) delete i.message.context.sources.version; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect( + received[0].data[integrationCasedString(integrations[index], "id")] + ).toEqual(i.message.messageId); + expect(received[1].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[1].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect( + received[1].data[integrationCasedString(integrations[index], "id")] + ).toEqual(i.message.messageId); }); - - it("should not set id based on recordId if event type is not track", () => { - let i = input("page"); - i.message.recordId = 42; - i.metadata = {sourceCategory: "cloud"}; - if (i.message.context.sources) delete i.message.context.sources.version; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[0].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect( - received[0].data[integrationCasedString(integrations[index], "id")] - ).toEqual(i.message.messageId); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "id") - ] - ).toEqual("string"); - }); + }); + + it("should not set id based on recordId if event type is not track", () => { + let i = input("page"); + i.message.recordId = 42; + i.metadata = { sourceCategory: "cloud" }; + if (i.message.context.sources) delete i.message.context.sources.version; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[0].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect( + received[0].data[integrationCasedString(integrations[index], "id")] + ).toEqual(i.message.messageId); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "id") + ] + ).toEqual("string"); }); - - it("should set id based on recordId when sourceCategory is cloud and has source version", () => { - let i = input("track"); - i.message.recordId = 42; - i.message.properties.recordId = "anotherRecordId"; - i.metadata = {sourceCategory: "cloud"}; - i.message.context.sources = {version: 1.12}; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "record_id") - ] - ).toEqual("string"); - expect( - received[0].data[ - integrationCasedString(integrations[index], "record_id") - ] - ).toBe("42"); - - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "id") - ] - ).toEqual("int"); - expect( - received[1].data[integrationCasedString(integrations[index], "id")] - ).toBe(42); - expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[1].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - }); + }); + + it("should set id based on recordId when sourceCategory is cloud and has source version", () => { + let i = input("track"); + i.message.recordId = 42; + i.message.properties.recordId = "anotherRecordId"; + i.metadata = { sourceCategory: "cloud" }; + i.message.context.sources = { version: 1.12 }; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "record_id") + ] + ).toEqual("string"); + expect( + received[0].data[ + integrationCasedString(integrations[index], "record_id") + ] + ).toBe("42"); + + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "id") + ] + ).toEqual("int"); + expect( + received[1].data[integrationCasedString(integrations[index], "id")] + ).toBe(42); + expect(received[1].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[1].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); }); - - it("should set id based on recordId when sourceCategory is singer-protocol and has source version", () => { - let i = input("track"); - i.message.recordId = 42; - i.message.properties.recordId = "anotherRecordId"; - i.metadata = {sourceCategory: "singer-protocol"}; - i.message.context.sources = {version: 1.12}; - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect( - received[0].metadata.columns[ - integrationCasedString(integrations[index], "record_id") - ] - ).toEqual("string"); - expect( - received[0].data[ - integrationCasedString(integrations[index], "record_id") - ] - ).toBe("42"); - - expect( - received[1].metadata.columns[ - integrationCasedString(integrations[index], "id") - ] - ).toEqual("int"); - expect( - received[1].data[integrationCasedString(integrations[index], "id")] - ).toBe(42); - expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - expect(received[1].data).not.toHaveProperty( - integrationCasedString(integrations[index], "record_id") - ); - }); + }); + + it("should set id based on recordId when sourceCategory is singer-protocol and has source version", () => { + let i = input("track"); + i.message.recordId = 42; + i.message.properties.recordId = "anotherRecordId"; + i.metadata = { sourceCategory: "singer-protocol" }; + i.message.context.sources = { version: 1.12 }; + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "record_id") + ] + ).toEqual("string"); + expect( + received[0].data[ + integrationCasedString(integrations[index], "record_id") + ] + ).toBe("42"); + + expect( + received[1].metadata.columns[ + integrationCasedString(integrations[index], "id") + ] + ).toEqual("int"); + expect( + received[1].data[integrationCasedString(integrations[index], "id")] + ).toBe(42); + expect(received[1].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); + expect(received[1].data).not.toHaveProperty( + integrationCasedString(integrations[index], "record_id") + ); }); - - it("should throw error when sourceCategory is cloud and has source version and recordId is missing", () => { - let i = input("track"); - i.message.properties.recordId = "anotherRecordId"; - i.metadata = {sourceCategory: "cloud"}; - i.message.context.sources = {version: 1.12}; - transformers.forEach((transformer, index) => { - expect(() => transformer.process(i)).toThrow( - "recordId cannot be empty for cloud sources events" - ); - }); + }); + + it("should throw error when sourceCategory is cloud and has source version and recordId is missing", () => { + let i = input("track"); + i.message.properties.recordId = "anotherRecordId"; + i.metadata = { sourceCategory: "cloud" }; + i.message.context.sources = { version: 1.12 }; + transformers.forEach((transformer, index) => { + expect(() => transformer.process(i)).toThrow( + "recordId cannot be empty for cloud sources events" + ); }); + }); }); describe("handle level three nested events from sources", () => { - it("should stringify event properties whose level ge 3 for cloud sources", () => { - const i = input("track"); - i.metadata = {sourceCategory: "cloud"}; - i.message.properties = Object.assign(i.message.properties, { - n0: { - prop0: "prop level 0", - n1: { - prop1: "prop level 1", - n2: { - prop2: "prop level 2", - n3: { - prop3: "prop level 3", - n4: {prop4: "prop level 4"} - } - } - } + it("should stringify event properties whose level ge 3 for cloud sources", () => { + const i = input("track"); + i.metadata = { sourceCategory: "cloud" }; + i.message.properties = Object.assign(i.message.properties, { + n0: { + prop0: "prop level 0", + n1: { + prop1: "prop level 1", + n2: { + prop2: "prop level 2", + n3: { + prop3: "prop level 3", + n4: { prop4: "prop level 4" } } - }); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") - ); - expect(received[1].data).not.toHaveProperty( - integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") - ); - }); + } + } + } }); - - it("should not stringify event properties whose level ge 3 for non cloud sources", () => { - const i = input("track"); - i.message.properties = Object.assign(i.message.properties, { - n0: { - prop0: "prop level 0", - n1: { - prop1: "prop level 1", - n2: { - prop2: "prop level 2", - n3: { - prop3: "prop level 3", - n4: {prop4: "prop level 4"} - } - } - } + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[1].metadata.columns).not.toHaveProperty( + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") + ); + expect(received[1].data).not.toHaveProperty( + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") + ); + }); + }); + + it("should not stringify event properties whose level ge 3 for non cloud sources", () => { + const i = input("track"); + i.message.properties = Object.assign(i.message.properties, { + n0: { + prop0: "prop level 0", + n1: { + prop1: "prop level 1", + n2: { + prop2: "prop level 2", + n3: { + prop3: "prop level 3", + n4: { prop4: "prop level 4" } } - }); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") - ); - expect(received[1].data).toHaveProperty( - integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") - ); - }); + } + } + } + }); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[1].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") + ); + expect(received[1].data).toHaveProperty( + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") + ); }); + }); }); describe("Handle no of columns in an event", () => { - it("should throw an error if no of columns are more than 200", () => { - const i = input("track"); - transformers - .filter((transformer, index) => integrations[index] !== "s3_datalake" && integrations[index] !== "gcs_datalake" && integrations[index] !== "azure_datalake") - .forEach((transformer, index) => { - i.message.properties = largeNoOfColumnsevent; - expect(() => transformer.process(i)).toThrow( - "transformer: Too many columns outputted from the event" - ); - }); - }); - - it("should not throw an error if no of columns are more than 200 and the event is from rudder-sources", () => { - const i = input("track"); - transformers.forEach((transformer, index) => { - i.message.channel = "sources"; - i.message.properties = largeNoOfColumnsevent; - expect(() => transformer.process(i)).not.toThrow(); - }); + it("should throw an error if no of columns are more than 200", () => { + const i = input("track"); + transformers + .filter((transformer, index) => integrations[index] !== "s3_datalake" && integrations[index] !== "gcs_datalake" && integrations[index] !== "azure_datalake") + .forEach((transformer, index) => { + i.message.properties = largeNoOfColumnsevent; + expect(() => transformer.process(i)).toThrow( + "transformer: Too many columns outputted from the event" + ); + }); + }); + + it("should not throw an error if no of columns are more than 200 and the event is from rudder-sources", () => { + const i = input("track"); + transformers.forEach((transformer, index) => { + i.message.channel = "sources"; + i.message.properties = largeNoOfColumnsevent; + expect(() => transformer.process(i)).not.toThrow(); }); + }); }); describe("Add auto generated messageId for events missing it", () => { - it("should remove context_ip set by user in properties if missing in event", () => { - eventTypes.forEach(evType => { - let i = input(evType); - delete i.message.messageId; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "id") - ); - expect(received[0].data).toHaveProperty( - integrationCasedString(integrations[index], "id") - ); - expect( - received[0].data[integrationCasedString(integrations[index], "id")] - ).toMatch(/auto-.*/); - }); - }); + it("should remove context_ip set by user in properties if missing in event", () => { + eventTypes.forEach(evType => { + let i = input(evType); + delete i.message.messageId; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "id") + ); + expect(received[0].data).toHaveProperty( + integrationCasedString(integrations[index], "id") + ); + expect( + received[0].data[integrationCasedString(integrations[index], "id")] + ).toMatch(/auto-.*/); + }); }); + }); }); describe("Add receivedAt for events missing it", () => { - it("should remove context_ip set by user in properties if missing in event", () => { - eventTypes.forEach(evType => { - let i = input(evType); - delete i.message.receivedAt; - delete i.message.received_at; - - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integrations[index], "received_at") - ); - expect(received[0].data).toHaveProperty( - integrationCasedString(integrations[index], "received_at") - ); - }); - }); + it("should remove context_ip set by user in properties if missing in event", () => { + eventTypes.forEach(evType => { + let i = input(evType); + delete i.message.receivedAt; + delete i.message.received_at; + + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received[0].metadata.columns).toHaveProperty( + integrationCasedString(integrations[index], "received_at") + ); + expect(received[0].data).toHaveProperty( + integrationCasedString(integrations[index], "received_at") + ); + }); }); + }); }); describe("Integration options", () => { - describe("track", () => { - it("should generate two events for every track call", () => { - const i = opInput("track"); - transformers.forEach((transformer, index) => { - const {jsonPaths} = i.destination.Config; - if (integrations[index] === "postgres") { - delete i.destination.Config.jsonPaths; - } - const received = transformer.process(i); - i.destination.Config.jsonPaths = jsonPaths; - expect(received).toEqual(opOutput("track", integrations[index])); - }); - }); + describe("track", () => { + it("should generate two events for every track call", () => { + const i = opInput("track"); + transformers.forEach((transformer, index) => { + const {jsonPaths} = i.destination.Config; + if (integrations[index] === "postgres") { + delete i.destination.Config.jsonPaths; + } + const received = transformer.process(i); + i.destination.Config.jsonPaths = jsonPaths; + expect(received).toEqual(opOutput("track", integrations[index])); + }); + }); + }); + + describe("users", () => { + it("should skip users when skipUsersTable is set", () => { + const i = opInput("users"); + transformers.forEach((transformer, index) => { + const received = transformer.process(i); + expect(received).toEqual(opOutput("users", integrations[index])); + }); }); + }); + + describe("json paths", () => { + const output = (config, provider) => { + switch (provider) { + case "rs": + return _.cloneDeep(config.output.rs); + case "bq": + return _.cloneDeep(config.output.bq); + case "postgres": + return _.cloneDeep(config.output.postgres); + case "snowflake": + return _.cloneDeep(config.output.snowflake); + default: + return _.cloneDeep(config.output.default); + } + } - describe("users", () => { - it("should skip users when skipUsersTable is set", () => { - const i = opInput("users"); - transformers.forEach((transformer, index) => { - const received = transformer.process(i); - expect(received).toEqual(opOutput("users", integrations[index])); - }); - }); + const testCases = [ + { + eventType: "aliases", + }, + { + eventType: "groups", + }, + { + eventType: "identifies", + }, + { + eventType: "pages", + }, + { + eventType: "screens", + }, + { + eventType: "tracks", + }, + { + eventType: "extract", + }, + ]; + + for (const testCase of testCases) { + transformers.forEach((transformer, index) => { + it(`new ${testCase.eventType} for ${integrations[index]}`, () => { + const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); + const input = _.cloneDeep(config.input); + const received = transformer.process(input); + expect(received).toEqual(output(config, integrations[index])); + }) + + it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { + const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); + const input = _.cloneDeep(config.input); + const received = transformer.process(input); + expect(received).toEqual(output(config, integrations[index])); + }) + }); + } + }); +}); + +describe("validTimestamp", () => { + const testCases = [ + { + name: "undefined input should return false", + input: undefined, + expected: false, + }, + { + name: "negative year and time input should return false #1", + input: '-0001-11-30T00:00:00+0000', + expected: false, + }, + { + name: "negative year and time input should return false #2", + input: '-2023-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "negative year and time input should return false #3", + input: '-1900-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "positive year and time input should return false", + input: '+2023-06-14T05:23:59.244Z', + expected: false, + }, + { + name: "valid timestamp input should return true", + input: '2023-06-14T05:23:59.244Z', + expected: true, + }, + { + name: "non-date string input should return false", + input: 'abc', + expected: false, + }, + { + name: "malicious string input should return false", + input: '%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216Windows%u2216win%u002ein', + expected: false, + }, + { + name: "empty string input should return false", + input: '', + expected: false, + }, + { + name: "valid date input should return true", + input: '2023-06-14', + expected: true, + }, + { + name: "time-only input should return false", + input: '05:23:59.244Z', + expected: false, + }, + { + name: "non-string input should return false", + input: { abc: 123 }, + expected: false, + }, + { + name: "object with toString method input should return false", + input: { + toString: '2023-06-14T05:23:59.244Z' + }, + expected: false, + }, + ]; + + for (const testCase of testCases) { + it(`should return ${testCase.expected} for ${testCase.name}`, () => { + expect(validTimestamp(testCase.input)).toEqual(testCase.expected); }); + } +}); - describe("json paths", () => { - const output = (config, provider) => { - switch (provider) { - case "rs": - return _.cloneDeep(config.output.rs); - case "bq": - return _.cloneDeep(config.output.bq); - case "postgres": - return _.cloneDeep(config.output.postgres); - case "snowflake": - return _.cloneDeep(config.output.snowflake); - default: - return _.cloneDeep(config.output.default); - } - } - const testCases = [ - { - eventType: "aliases", - }, - { - eventType: "groups", - }, - { - eventType: "identifies", - }, - { - eventType: "pages", - }, - { - eventType: "screens", - }, - { - eventType: "tracks", - }, - { - eventType: "extract", - }, - ]; - for (const testCase of testCases) { - transformers.forEach((transformer, index) => { - it(`new ${testCase.eventType} for ${integrations[index]}`, () => { - const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); - const input = _.cloneDeep(config.input); - const received = transformer.process(input); - expect(received).toEqual(output(config, integrations[index])); - }) - - it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { - const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); - const input = _.cloneDeep(config.input); - const received = transformer.process(input); - expect(received).toEqual(output(config, integrations[index])); - }) - }); - } +describe("isBlank", () => { + const testCases = [ + { + name: "null", + input: null, + expected: true + }, + { + name: "empty string", + input: "", + expected: true + }, + { + name: "non-empty string", + input: "test", + expected: false + }, + { + name: "numeric value", + input: 1634762544, + expected: false + }, + { + name: "object with toString property", + input: { + toString: '2023-06-14T05:23:59.244Z' + }, + expected: false + }, + ]; + + for (const testCase of testCases) { + it(`should return ${testCase.expected} for ${testCase.name}`, () => { + expect(isBlank(testCase.input)).toEqual(testCase.expected); }); + } }); describe("Destination config", () => {