From f44f1488acb05bb5096e5f9d7bade06f76f55d13 Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Fri, 6 Dec 2024 14:05:11 +0530 Subject: [PATCH 1/4] chore: add metrics for mailjet (#3917) --- src/util/prometheus.js | 6 ++++++ src/v0/destinations/mailjet/transform.js | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/src/util/prometheus.js b/src/util/prometheus.js index c8dd55068b..a4c12e4ea8 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -520,6 +520,12 @@ class Prometheus { type: 'counter', labelNames: ['destination_id'], }, + { + name: 'mailjet_packing_size', + help: 'mailjet_packing_size', + type: 'gauge', + labelNames: ['group'], + }, { name: 'hs_batch_size', help: 'hs_batch_size', diff --git a/src/v0/destinations/mailjet/transform.js b/src/v0/destinations/mailjet/transform.js index 78b4f766d1..0a742b4cf7 100644 --- a/src/v0/destinations/mailjet/transform.js +++ b/src/v0/destinations/mailjet/transform.js @@ -1,5 +1,6 @@ const lodash = require('lodash'); const { TransformationError, InstrumentationError } = require('@rudderstack/integrations-lib'); +const stats = require('../../../util/stats'); const { getSuccessRespEvents, defaultRequestConfig, @@ -105,6 +106,9 @@ const batchEvents = (successRespList) => { const eventChunks = lodash.chunk(eventGroups[combination], MAX_BATCH_SIZE); // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] eventChunks.forEach((chunk) => { + stats.gauge('mailjet_packing_size', chunk.length, { + group: combination, + }); const batchEventResponse = generateBatchedPaylaodForArray(chunk, combination); batchedResponseList.push( getSuccessRespEvents( From 313710ca725538e5ffe357216d9c88e444f995c8 Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:04:06 +0530 Subject: [PATCH 2/4] fix: remove redundant ids and userIdentifier when gbraid or wbraid are there (#3910) * fix: remove redundant ids and userIdentifier when gbraid or wbraid are there * chore: address comment * chore: fix sonar cloud issue * chore: fix sonar cloud issue * chore: decrease Cognitive Complexity from 17 to the 15 * chore: fix lint error * chore: specify head commit hash for checkout temporarily * chore: fix fetch depth * chore: fix fetch depth to 0 * chore: reset commit sha --------- Co-authored-by: Sai Kumar Battinoju --- .../dt-test-and-report-code-coverage.yml | 2 +- .../transform.js | 4 + .../utils.js | 62 +++- .../utils.test.js | 4 +- .../processor/data.ts | 287 ++++++++++++++++-- .../router/data.ts | 2 - 6 files changed, 315 insertions(+), 46 deletions(-) diff --git a/.github/workflows/dt-test-and-report-code-coverage.yml b/.github/workflows/dt-test-and-report-code-coverage.yml index 2c76898882..12bd9ac78b 100644 --- a/.github/workflows/dt-test-and-report-code-coverage.yml +++ b/.github/workflows/dt-test-and-report-code-coverage.yml @@ -33,7 +33,7 @@ jobs: - name: Checkout uses: actions/checkout@v4.2.1 with: - fetch-depth: 1 + fetch-depth: 0 - name: Setup Node uses: actions/setup-node@v4.0.4 diff --git a/src/v0/destinations/google_adwords_offline_conversions/transform.js b/src/v0/destinations/google_adwords_offline_conversions/transform.js index 76b12587cd..2648f03e8a 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/transform.js +++ b/src/v0/destinations/google_adwords_offline_conversions/transform.js @@ -18,6 +18,7 @@ const { getClickConversionPayloadAndEndpoint, getConsentsDataFromIntegrationObj, getCallConversionPayload, + updateConversion, } = require('./utils'); const helper = require('./helper'); @@ -48,6 +49,9 @@ const getConversions = (message, metadata, { Config }, event, conversionType) => filteredCustomerId, eventLevelConsentsData, ); + convertedPayload.payload.conversions[0] = updateConversion( + convertedPayload.payload.conversions[0], + ); payload = convertedPayload.payload; endpoint = convertedPayload.endpoint; } else if (conversionType === 'store') { diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js index 89fe609df9..2d47095eea 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.js @@ -317,6 +317,35 @@ const getStoreConversionPayload = (message, Config, event) => { return payload; }; +const hasClickId = (conversion) => { + const { gbraid, wbraid, gclid } = conversion; + return gclid || wbraid || gbraid; +}; +const populateUserIdentifier = ({ email, phone, properties, payload, UserIdentifierSource }) => { + const copiedPayload = cloneDeep(payload); + // userIdentifierSource + // if userIdentifierSource doesn't exist in properties + // then it is taken from the webapp config + if (!properties.userIdentifierSource && UserIdentifierSource !== 'none') { + set( + copiedPayload, + 'conversions[0].userIdentifiers[0].userIdentifierSource', + UserIdentifierSource, + ); + // one of email or phone must be provided when none of gclid, wbraid and gbraid provided + } + if (!email && !phone) { + if (!hasClickId(copiedPayload.conversions[0])) { + throw new InstrumentationError( + `Either an email address or a phone number is required for user identification when none of gclid, wbraid, or gbraid is provided.`, + ); + } else { + // we are deleting userIdentifiers if any one of gclid, wbraid and gbraid is there but email or phone is not present + delete copiedPayload.conversions[0].userIdentifiers; + } + } + return copiedPayload; +}; const getClickConversionPayloadAndEndpoint = ( message, Config, @@ -335,7 +364,7 @@ const getClickConversionPayloadAndEndpoint = ( updatedClickMapping = removeHashToSha256TypeFromMappingJson(updatedClickMapping); } - const payload = constructPayload(message, updatedClickMapping); + let payload = constructPayload(message, updatedClickMapping); const endpoint = CLICK_CONVERSION.replace(':customerId', filteredCustomerId); @@ -353,17 +382,8 @@ const getClickConversionPayloadAndEndpoint = ( set(payload, 'conversions[0].cartData.items', itemList); } - // userIdentifierSource - // if userIdentifierSource doesn't exist in properties - // then it is taken from the webapp config - if (!properties.userIdentifierSource && UserIdentifierSource !== 'none') { - set(payload, 'conversions[0].userIdentifiers[0].userIdentifierSource', UserIdentifierSource); + payload = populateUserIdentifier({ email, phone, properties, payload, UserIdentifierSource }); - // one of email or phone must be provided - if (!email && !phone) { - throw new InstrumentationError(`Either of email or phone is required for user identifier`); - } - } // either of email or phone should be passed // defaultUserIdentifier depends on the webapp configuration // Ref - https://developers.google.com/google-ads/api/rest/reference/rest/v11/customers/uploadClickConversions#ClickConversion @@ -411,6 +431,25 @@ const getConsentsDataFromIntegrationObj = (message) => { return integrationObj?.consents || {}; }; +/** + * remove redundant ids + * @param {*} conversionCopy + */ +const updateConversion = (conversion) => { + const conversionCopy = cloneDeep(conversion); + if (conversionCopy.gclid) { + delete conversionCopy.wbraid; + delete conversionCopy.gbraid; + } + if (conversionCopy.wbraid && conversionCopy.gbraid) { + throw new InstrumentationError(`You can't use both wbraid and gbraid.`); + } + if (conversionCopy.wbraid || conversionCopy.gbraid) { + delete conversionCopy.userIdentifiers; + } + return conversionCopy; +}; + module.exports = { validateDestinationConfig, generateItemListFromProducts, @@ -423,4 +462,5 @@ module.exports = { getExisitingUserIdentifier, getConsentsDataFromIntegrationObj, getCallConversionPayload, + updateConversion, }; diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js index 2d1863413c..b6c6653782 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js @@ -244,7 +244,9 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { }; expect(() => getClickConversionPayloadAndEndpoint(fittingPayload, config, '9625812972'), - ).toThrow('Either of email or phone is required for user identifier'); + ).toThrow( + 'Either an email address or a phone number is required for user identification when none of gclid, wbraid, or gbraid is provided.', + ); }); it('finaliseConsent', () => { diff --git a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts index ab3e19dc2f..3ecae2b92d 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts @@ -231,8 +231,6 @@ export const data = [ JSON: { conversions: [ { - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -528,8 +526,6 @@ export const data = [ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', }, - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -821,8 +817,6 @@ export const data = [ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', }, - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -1911,7 +1905,6 @@ export const data = [ userId: '12345', properties: { gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', conversionCustomVariable: 'conversionCustomVariable', @@ -1935,7 +1928,6 @@ export const data = [ ], userIdentifierSource: 'FIRST_PARTY', conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: '1', currency: 'GBP', @@ -2052,7 +2044,6 @@ export const data = [ ], properties: { gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', conversionCustomVariable: 'conversionCustomVariable', @@ -2076,7 +2067,6 @@ export const data = [ ], userIdentifierSource: 'FIRST_PARTY', conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: '1', currency: 'GBP', @@ -2093,7 +2083,6 @@ export const data = [ adUserData: 'UNSPECIFIED', }, gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -2110,15 +2099,7 @@ export const data = [ }, ], }, - userIdentifiers: [ - { - userIdentifierSource: 'FIRST_PARTY', - hashedPhoneNumber: - '04e1dabb7c1348b72bfa87da179c9697c69af74827649266a5da8cdbb367abcd', - }, - ], conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: 1, currencyCode: 'GBP', @@ -2169,7 +2150,6 @@ export const data = [ ], properties: { gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', conversionCustomVariable: 'conversionCustomVariable', @@ -2193,7 +2173,6 @@ export const data = [ ], userIdentifierSource: 'FIRST_PARTY', conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: '1', currency: 'GBP', @@ -3751,20 +3730,73 @@ export const data = [ metadata: { secret: { access_token: 'abcd1234', - refresh_token: 'efgh5678', developer_token: 'ijkl91011', + refresh_token: 'efgh5678', }, }, - statusCode: 400, - error: 'Either of email or phone is required for user identifier', - statTags: { - errorCategory: 'dataValidation', - errorType: 'instrumentation', - destType: 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS', - module: 'destination', - implementation: 'native', - feature: 'processor', + output: { + body: { + FORM: {}, + JSON: { + conversions: [ + { + cartData: { + items: [ + { + productId: '123445', + quantity: 123, + }, + ], + }, + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + conversionDateTime: '2022-01-01 12:32:45-08:00', + conversionEnvironment: 'WEB', + gclid: 'gclid', + }, + ], + partialFailure: true, + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + files: {}, + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + 'login-customer-id': '8617859087', + }, + method: 'POST', + params: { + customVariables: [ + { + from: 'value', + to: 'revenue', + }, + { + from: 'total', + to: 'cost', + }, + ], + customerId: '9625812972', + event: 'Sign-up - click', + properties: { + conversionDateTime: '2022-01-01 12:32:45-08:00', + gclid: 'gclid', + product_id: '123445', + quantity: 123, + }, + }, + type: 'REST', + userId: '', + version: '1', }, + statusCode: 200, }, ], }, @@ -5704,4 +5736,197 @@ export const data = [ }, mockFns: timestampMock, }, + { + name: 'google_adwords_offline_conversions', + description: 'Test 28 : when both gbraid and wbraid are available', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + id: '0572f78fa49c648e', + name: 'generic_x86_arm', + type: 'Android', + model: 'AOSP on IA Emulator', + manufacturer: 'Google', + adTrackingEnabled: true, + advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8', + }, + traits: { + phone: 'alex@example.com', + firstName: 'John', + lastName: 'Gomes', + city: 'London', + state: 'England', + countryCode: 'GB', + postalCode: 'EC3M', + streetAddress: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + }, + event: 'Promotion Clicked', + type: 'track', + messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', + originalTimestamp: '2019-10-14T11:15:18.299Z', + anonymousId: '00000000000000000000000000', + userId: '12345', + properties: { + gbraid: 'gbraid', + wbraid: 'wbraid', + externalAttributionCredit: 10, + externalAttributionModel: 'externalAttributionModel', + conversionCustomVariable: 'conversionCustomVariable', + value: 'value', + merchantId: '9876merchantId', + feedCountryCode: 'feedCountryCode', + feedLanguageCode: 'feedLanguageCode', + localTransactionCost: 20, + products: [ + { + product_id: '507f1f77bcf86cd799439011', + quantity: '2', + price: '50', + sku: '45790-32', + name: 'Monopoly: 3rd Edition', + position: '1', + category: 'cars', + url: 'https://www.example.com/product/path', + image_url: 'https://www.example.com/product/path.jpg', + }, + ], + userIdentifierSource: 'FIRST_PARTY', + conversionEnvironment: 'WEB', + conversionValue: '1', + currency: 'GBP', + orderId: 'PL-123QR', + }, + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + customerId: '962-581-2972', + subAccount: false, + eventsToOfflineConversionsTypeMapping: [ + { + from: 'Sign up completed', + to: 'click', + }, + { + from: 'Download', + to: 'call', + }, + { + from: 'Promotion Clicked', + to: 'click', + }, + { + from: 'Product Searched', + to: 'call', + }, + ], + eventsToConversionsNamesMapping: [ + { + from: 'Sign up completed', + to: 'Sign-up - click', + }, + { + from: 'Download', + to: 'Page view', + }, + { + from: 'Promotion Clicked', + to: 'Sign-up - click', + }, + { + from: 'Product Searched', + to: 'search', + }, + ], + customVariables: [ + { + from: 'value', + to: 'revenue', + }, + { + from: 'total', + to: 'cost', + }, + ], + UserIdentifierSource: 'THIRD_PARTY', + conversionEnvironment: 'WEB', + hashUserIdentifier: true, + defaultUserIdentifier: 'email', + validateOnly: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: "You can't use both wbraid and gbraid.", + metadata: { + secret: { + access_token: 'abcd1234', + developer_token: 'ijkl91011', + refresh_token: 'efgh5678', + }, + }, + statTags: { + destType: 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + mockFns: timestampMock, + }, ]; diff --git a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts index 9d1ba220c8..82c8e9b3ff 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts @@ -738,8 +738,6 @@ export const data = [ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', }, - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', From 12621c8eee641f5a03a997e95ed016cff0eefde7 Mon Sep 17 00:00:00 2001 From: Akash Chetty Date: Mon, 9 Dec 2024 10:36:42 +0530 Subject: [PATCH 3/4] fix: skipping users events for snowpipe streaming (#3836) --- .../snowpipe_streaming/transform.js | 27 +- src/warehouse/config/ReservedKeywords.json | 93 ++++ src/warehouse/index.js | 18 +- src/warehouse/v1/util.js | 4 +- test/__tests__/data/warehouse/events.js | 501 ++++++++++++++++++ .../warehouse/integration_options_events.js | 166 +++++- .../integrations/jsonpaths/legacy/aliases.js | 73 +++ .../integrations/jsonpaths/legacy/extract.js | 45 ++ .../integrations/jsonpaths/legacy/groups.js | 73 +++ .../jsonpaths/legacy/identifies.js | 89 ++++ .../integrations/jsonpaths/legacy/pages.js | 69 +++ .../integrations/jsonpaths/legacy/screens.js | 69 +++ .../integrations/jsonpaths/legacy/tracks.js | 151 ++++++ .../integrations/jsonpaths/new/aliases.js | 70 +++ .../integrations/jsonpaths/new/extract.js | 45 ++ .../integrations/jsonpaths/new/groups.js | 70 +++ .../integrations/jsonpaths/new/identifies.js | 89 ++++ .../integrations/jsonpaths/new/pages.js | 66 +++ .../integrations/jsonpaths/new/screens.js | 66 +++ .../integrations/jsonpaths/new/tracks.js | 151 ++++++ test/__tests__/data/warehouse/names.js | 39 ++ test/__tests__/warehouse.test.js | 285 +++++----- 22 files changed, 2120 insertions(+), 139 deletions(-) diff --git a/src/v0/destinations/snowpipe_streaming/transform.js b/src/v0/destinations/snowpipe_streaming/transform.js index 3b0a6e1a93..36fe19ceff 100644 --- a/src/v0/destinations/snowpipe_streaming/transform.js +++ b/src/v0/destinations/snowpipe_streaming/transform.js @@ -1 +1,26 @@ -module.exports = require('../snowflake/transform'); +const transform = require('../snowflake/transform'); +const { processWarehouseMessage } = require('../../../warehouse'); + +const provider = 'snowpipe_streaming'; + +function process(event) { + const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; + const whStoreEvent = event.destination.Config.storeFullEvent === true; + const destJsonPaths = event.destination?.Config?.jsonPaths || ''; + return processWarehouseMessage(event.message, { + metadata: event.metadata, + whSchemaVersion, + whStoreEvent, + getDataTypeOverride: transform.getDataTypeOverride, + provider, + sourceCategory: event.metadata ? event.metadata.sourceCategory : null, + destJsonPaths, + destConfig: event.destination?.Config, + }); +} + +module.exports = { + provider, + process, + getDataTypeOverride: transform.getDataTypeOverride, +}; diff --git a/src/warehouse/config/ReservedKeywords.json b/src/warehouse/config/ReservedKeywords.json index 3c383d24bf..53b8738e1d 100644 --- a/src/warehouse/config/ReservedKeywords.json +++ b/src/warehouse/config/ReservedKeywords.json @@ -1519,6 +1519,99 @@ "WHERE": true, "WITH": true }, + "SNOWPIPE_STREAMING": { + "ACCOUNT": true, + "ALL": true, + "ALTER": true, + "AND": true, + "ANY": true, + "AS": true, + "BETWEEN": true, + "BY": true, + "CASE": true, + "CAST": true, + "CHECK": true, + "COLUMN": true, + "CONNECT": true, + "CONNECTION": true, + "CONSTRAINT": true, + "CREATE": true, + "CROSS": true, + "CURRENT": true, + "CURRENT_DATE": true, + "CURRENT_TIME": true, + "CURRENT_TIMESTAMP": true, + "CURRENT_USER": true, + "DATABASE": true, + "DELETE": true, + "DISTINCT": true, + "DROP": true, + "ELSE": true, + "EXISTS": true, + "FALSE": true, + "FOLLOWING": true, + "FOR": true, + "FROM": true, + "FULL": true, + "GRANT": true, + "GROUP": true, + "GSCLUSTER": true, + "HAVING": true, + "ILIKE": true, + "IN": true, + "INCREMENT": true, + "INNER": true, + "INSERT": true, + "INTERSECT": true, + "INTO": true, + "IS": true, + "ISSUE": true, + "JOIN": true, + "LATERAL": true, + "LEFT": true, + "LIKE": true, + "LOCALTIME": true, + "LOCALTIMESTAMP": true, + "MINUS": true, + "NATURAL": true, + "NOT": true, + "NULL": true, + "OF": true, + "ON": true, + "OR": true, + "ORDER": true, + "ORGANIZATION": true, + "QUALIFY": true, + "REGEXP": true, + "REVOKE": true, + "RIGHT": true, + "RLIKE": true, + "ROW": true, + "ROWS": true, + "SAMPLE": true, + "SCHEMA": true, + "SELECT": true, + "SET": true, + "SOME": true, + "START": true, + "TABLE": true, + "TABLESAMPLE": true, + "THEN": true, + "TO": true, + "TRIGGER": true, + "TRUE": true, + "TRY_CAST": true, + "UNION": true, + "UNIQUE": true, + "UPDATE": true, + "USING": true, + "VALUES": true, + "VIEW": true, + "WHEN": true, + "WHENEVER": true, + "WHERE": true, + "WITH": true + }, "CLICKHOUSE": {}, "S3_DATALAKE": { "ALL": true, diff --git a/src/warehouse/index.js b/src/warehouse/index.js index 3491a257da..ea663c9b2f 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -349,7 +349,10 @@ function stringLikeObjectToString(obj) { */ function getColumns(options, event, columnTypes) { const columns = {}; - const uuidTS = options.provider === 'snowflake' ? 'UUID_TS' : 'uuid_ts'; + const uuidTS = + options.provider === 'snowflake' || options.provider === 'snowpipe_streaming' + ? 'UUID_TS' + : 'uuid_ts'; columns[uuidTS] = 'datetime'; // add loaded_at for bq to be segment compatible if (options.provider === 'bq') { @@ -377,6 +380,7 @@ function getColumns(options, event, columnTypes) { const fullEventColumnTypeByProvider = { snowflake: 'json', + snowpipe_streaming: 'json', rs: 'text', bq: 'string', postgres: 'json', @@ -613,6 +617,15 @@ function enhanceContextWithSourceDestInfo(message, metadata) { message.context = context; } +function shouldSkipUsersTable(options) { + return ( + options.provider === 'snowpipe_streaming' || + options.destConfig?.skipUsersTable || + options.integrationOptions?.skipUsersTable || + false + ); +} + function processWarehouseMessage(message, options) { const utils = getVersionedUtils(options.whSchemaVersion); options.utils = utils; @@ -638,8 +651,7 @@ function processWarehouseMessage(message, options) { const eventType = message.type?.toLowerCase(); const skipTracksTable = options.destConfig?.skipTracksTable || options.integrationOptions.skipTracksTable || false; - const skipUsersTable = - options.destConfig?.skipUsersTable || options.integrationOptions.skipUsersTable || false; + const skipUsersTable = shouldSkipUsersTable(options); const skipReservedKeywordsEscaping = options.integrationOptions.skipReservedKeywordsEscaping || false; diff --git a/src/warehouse/v1/util.js b/src/warehouse/v1/util.js index d1289bc674..95dd59b030 100644 --- a/src/warehouse/v1/util.js +++ b/src/warehouse/v1/util.js @@ -11,7 +11,7 @@ function safeTableName(options, name = '') { if (tableName === '') { throw new TransformationError('Table name cannot be empty.'); } - if (provider === 'snowflake') { + if (provider === 'snowflake' || provider === 'snowpipe_streaming') { tableName = tableName.toUpperCase(); } else if (provider === 'postgres') { tableName = tableName.substr(0, 63); @@ -41,7 +41,7 @@ function safeColumnName(options, name = '') { if (columnName === '') { throw new TransformationError('Column name cannot be empty.'); } - if (provider === 'snowflake') { + if (provider === 'snowflake' || provider === 'snowpipe_streaming') { columnName = columnName.toUpperCase(); } else if (provider === 'postgres') { columnName = columnName.substr(0, 63); diff --git a/test/__tests__/data/warehouse/events.js b/test/__tests__/data/warehouse/events.js index bca6f776be..99343e43e1 100644 --- a/test/__tests__/data/warehouse/events.js +++ b/test/__tests__/data/warehouse/events.js @@ -368,6 +368,146 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "TRACKS", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "button clicked", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + EVENT: "button_clicked" + } + }, + { + metadata: { + table: "BUTTON_CLICKED", + columns: { + UUID_TS: "datetime", + CURRENCY: "string", + REVENUE: "int", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CURRENCY: "USD", + REVENUE: 50, + STACK_HISTORY_ERROR_DETAILS: [ + { + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" + } + ], + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "button clicked", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + EVENT: "button_clicked", + } + } + ], s3_datalake: [ { metadata: { @@ -845,6 +985,78 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "IDENTIFIES", + columns: { + UUID_TS: "datetime", + CITY: "string", + COUNTRY: "string", + EMAIL: "string", + FIRSTNAME: "string", + LASTNAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CITY: "Disney", + COUNTRY: "UK", + EMAIL: "mickey@disney.com", + FIRSTNAME: "Mickey", + LASTNAME: "Mouse", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "2536eda4-d638-4c93-8014-8ffe3f083214", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.363Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.362Z", + CHANNEL: "web" + } + }, + ], s3_datalake: [ { metadata: { @@ -1170,6 +1382,66 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "PAGES", + columns: { + UUID_TS: "datetime", + PATH: "string", + URL: "string", + NAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_USER_AGENT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + PATH: "/tests/html/index2.html", + URL: "http://localhost/tests/html/index2.html", + NAME: "sample title", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "dd266c67-9199-4a52-ba32-f46ddde67312", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.359Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.358Z", + CHANNEL: "web" + } + } + ], s3_datalake: [ { metadata: { @@ -1424,6 +1696,67 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "SCREENS", + columns: { + UUID_TS: "datetime", + PATH: "string", + CATEGORY: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + TIMESTAMP: "datetime", + CHANNEL: "string", + NAME: "string" + } + }, + data: { + PATH: "/abc", + CATEGORY: "test-category", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.0", + CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.0", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + CONTEXT_LOCALE: "en-US", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + ID: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + ANONYMOUS_ID: "00000000000000000000000000", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2019-10-14T11:15:53.296Z", + ORIGINAL_TIMESTAMP: "2019-10-14T11:15:18.299Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + CHANNEL: "web", + NAME: "ApplicationLoaded" + } + } + ], s3_datalake: [ { metadata: { @@ -1687,6 +2020,73 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "GROUPS", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_LOCALE: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_OS_ID: "string", + CONTEXT_OS_MANUFACTURER: "string", + CONTEXT_OS_MODEL: "string", + CONTEXT_OS_TYPE: "string", + CONTEXT_SCREEN_DENSITY: "int", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + GROUP_ID: "string", + SENT_AT: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + TIMESTAMP: "datetime", + CHANNEL: "string" + } + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.0", + CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.0", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + CONTEXT_LOCALE: "en-US", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_OS_ID: "72e528f869711c3d", + CONTEXT_OS_MANUFACTURER: "Google", + CONTEXT_OS_MODEL: "sdk_gphone_x86", + CONTEXT_OS_NAME: "", + CONTEXT_OS_TOKEN: "", + CONTEXT_OS_TYPE: "android", + CONTEXT_SCREEN_DENSITY: 2, + ID: "84e26acc-56a5-4835-8233-591137fca468", + ANONYMOUS_ID: "00000000000000000000000000", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + GROUP_ID: "g1", + SENT_AT: "2019-10-14T09:03:22.563Z", + ORIGINAL_TIMESTAMP: "2019-10-14T09:03:17.562Z", + TIMESTAMP: "2020-01-24T11:59:02.403Z", + RECEIVED_AT: "2020-01-24T11:59:02.403Z", + CHANNEL: "web" + } + } + ], s3_datalake: [ { metadata: { @@ -1956,6 +2356,70 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "ALIASES", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + PREVIOUS_ID: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "79313729-7fe5-4204-963a-dc46f4205e4e", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "1234abc", + SENT_AT: "2020-01-24T06:29:02.366Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.366Z", + CHANNEL: "web", + PREVIOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" + } + } + ], s3_datalake: [ { metadata: { @@ -2147,6 +2611,40 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "EXTRACT_EVENT", + columns: { + DATE_PROPERTY: "datetime", + DATE_PROPERTY_2: "datetime", + CONTEXT_SOURCES_JOB_ID: "string", + CONTEXT_SOURCES_JOB_RUN_ID: "string", + CONTEXT_SOURCES_TASK_RUN_ID: "string", + CONTEXT_SOURCES_VERSION: "string", + EVENT: "string", + ID: "string", + USER_ID: "string", + RECEIVED_AT: "datetime", + BOOLEAN_PROPERTY: "boolean", + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + DATE_PROPERTY: "2023-02-01T07:53:31.430Z", + DATE_PROPERTY_2: "2023-01-31T07:39:10.002Z", + CONTEXT_SOURCES_JOB_ID: "2JABSy1nq89H7xeJimBL2pCtOOp", + CONTEXT_SOURCES_JOB_RUN_ID: "cfd6705nsevh5p2l77ag", + CONTEXT_SOURCES_TASK_RUN_ID: "cfd6705nsevh5p2l77b0", + CONTEXT_SOURCES_VERSION: "version", + ID: "some-uuid", + USER_ID: "dummy-user-id-inside-properties", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + EVENT: "extract_event", + BOOLEAN_PROPERTY: true, + } + } + ], s3_datalake: [ { metadata: { @@ -2193,6 +2691,9 @@ function output(eventType, provider) { if (provider === "snowflake") { return _.cloneDeep(sampleEvents[eventType].output.snowflake); } + if (provider === "snowpipe_streaming") { + return _.cloneDeep(sampleEvents[eventType].output.snowpipe_streaming); + } if (provider === "s3_datalake") { return _.cloneDeep(sampleEvents[eventType].output.s3_datalake); } diff --git a/test/__tests__/data/warehouse/integration_options_events.js b/test/__tests__/data/warehouse/integration_options_events.js index 35eafb6a9b..b3f076851b 100644 --- a/test/__tests__/data/warehouse/integration_options_events.js +++ b/test/__tests__/data/warehouse/integration_options_events.js @@ -74,6 +74,13 @@ const sampleEvents = { jsonPaths: ["tMap"] } }, + SNOWPIPE_STREAMING: { + options: { + skipTracksTable: true, + useBlendoCasing: true, + jsonPaths: ["tMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -432,6 +439,88 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "_GROUPS", + columns: { + UUID_TS: "datetime", + CURRENCY: "string", + REVENUE: "int", + PATH_TO_$1_000_000: "string", + _9OMEGA: "boolean", + CAMELCASE123KEY: "string", + TESTMAP_NESTEDMAP: "json", + TMAP: "json", + TESTARRAY: "json", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USERAGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CURRENCY: "USD", + REVENUE: 50, + PATH_TO_$1_000_000: "None", + _9OMEGA: true, + CAMELCASE123KEY: "camel case", + TESTMAP_NESTEDMAP: '{"n1":"nested prop 1"}', + TMAP: '{"t1":10,"t2":20}', + TESTARRAY: '["This is","an","array"]', + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USERAGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "groups", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + EVENT: "groups" + } + } + ], s3_datalake: [ { metadata: { @@ -730,7 +819,7 @@ const sampleEvents = { ] } }, - users: { + identify: { input: { destination: { Config: { @@ -1255,6 +1344,75 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + ], s3_datalake: [ { "data": { @@ -1639,6 +1797,8 @@ function opOutput(eventType, provider) { switch (provider) { case "snowflake": return _.cloneDeep(sampleEvents[eventType].output.snowflake); + case "snowpipe_streaming": + return _.cloneDeep(sampleEvents[eventType].output.snowpipe_streaming); case "s3_datalake": return _.cloneDeep(sampleEvents[eventType].output.s3_datalake); case "rs": @@ -1646,13 +1806,13 @@ function opOutput(eventType, provider) { case "bq": return _.cloneDeep(sampleEvents[eventType].output.bq); case "gcs_datalake": - if (eventType === 'users') { + if (eventType === 'identify') { return _.cloneDeep(sampleEvents[eventType].output.gcs_datalake); } else { return _.cloneDeep(sampleEvents[eventType].output.default); } case "azure_datalake": - if (eventType === 'users') { + if (eventType === 'identify') { return _.cloneDeep(sampleEvents[eventType].output.azure_datalake); } else { return _.cloneDeep(sampleEvents[eventType].output.default); diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js index 30cce51fb7..7615abe5e1 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js @@ -78,6 +78,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -426,5 +434,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "ALIASES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js index 9d38ecc292..b28b3a161e 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js @@ -59,6 +59,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -276,6 +284,43 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_SOURCES_JOB_ID": "djfhksdjhfkjdhfkjahkf", + "CONTEXT_SOURCES_JOB_RUN_ID": "job_run_id", + "CONTEXT_SOURCES_TASK_RUN_ID": "task_run_id", + "CONTEXT_SOURCES_VERSION": "1169/merge", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EVENT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "P_MAP_NESTED_MAP_N_1": "nested prop 1", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50 + }, + "metadata": { + "columns": { + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_SOURCES_JOB_ID": "string", + "CONTEXT_SOURCES_JOB_RUN_ID": "string", + "CONTEXT_SOURCES_TASK_RUN_ID": "string", + "CONTEXT_SOURCES_VERSION": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EVENT": "string", + "ID": "string", + "P_MAP_NESTED_MAP_N_1": "string", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js index bbae993b27..457d251a74 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js @@ -78,6 +78,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, GCS_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -431,5 +439,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "GROUPS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js index 42f5cccf49..d91ef5f44b 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js @@ -53,6 +53,16 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "CTMap.nestedMap", + "TMap.nestedMap", + "CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -879,6 +889,85 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP_N_1": "nested prop 1", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "CT_MAP_NESTED_MAP_N_1": "nested prop 1", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "T_MAP_NESTED_MAP_N_1": "nested prop 1", + "UP_MAP_NESTED_MAP_N_1": "nested prop 1", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "CT_MAP_NESTED_MAP_N_1": "string", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "T_MAP_NESTED_MAP_N_1": "string", + "UP_MAP_NESTED_MAP_N_1": "string", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + ], datalake: [ { "data": { diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js index dc2d3a3318..322ee8f3e6 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js @@ -67,6 +67,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -404,5 +412,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PAGES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js index 9b789d2747..27d7b9a69b 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js @@ -67,6 +67,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + ".ctestMap.cnestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -404,5 +412,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "SCREENS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js index 691ce57ca1..aabfce149b 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js @@ -78,6 +78,15 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -828,6 +837,148 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "TRACKS" + } + }, + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EMAIL": "test@gmail.com", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EMAIL": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js index d5d57f85b9..4b74fd3768 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js @@ -66,6 +66,11 @@ module.exports = { jsonPaths: ["alias.traits.testMap.nestedMap", "alias.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["alias.traits.testMap.nestedMap", "alias.context.ctestMap.cnestedMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -414,5 +419,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "ALIASES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js index a950b7f526..03745c0502 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js @@ -59,6 +59,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "extract.properties.PMap.nestedMap", + "extract.context.CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -276,6 +284,43 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_SOURCES_JOB_ID": "djfhksdjhfkjdhfkjahkf", + "CONTEXT_SOURCES_JOB_RUN_ID": "job_run_id", + "CONTEXT_SOURCES_TASK_RUN_ID": "task_run_id", + "CONTEXT_SOURCES_VERSION": "1169/merge", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EVENT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50 + }, + "metadata": { + "columns": { + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_SOURCES_JOB_ID": "string", + "CONTEXT_SOURCES_JOB_RUN_ID": "string", + "CONTEXT_SOURCES_TASK_RUN_ID": "string", + "CONTEXT_SOURCES_VERSION": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EVENT": "string", + "ID": "string", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js index 6979aa1100..1e7c8110da 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js @@ -66,6 +66,11 @@ module.exports = { jsonPaths: ["group.traits.testMap.nestedMap", "group.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["group.traits.testMap.nestedMap", "group.context.ctestMap.cnestedMap"] + } + }, GCS_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -419,5 +424,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "GROUPS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js index 89fcc23cd5..d9c3b80df3 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js @@ -53,6 +53,16 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "identify.userProperties.UPMap.nestedMap", + "identify.context.traits.CTMap.nestedMap", + "identify.traits.TMap.nestedMap", + "identify.context.CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -879,6 +889,85 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "CT_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "T_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP": "json", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "CT_MAP_NESTED_MAP": "json", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "T_MAP_NESTED_MAP": "json", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + ], datalake: [ { "data": { diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js index d5282c1cfd..1f72ec31cf 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js @@ -55,6 +55,11 @@ module.exports = { jsonPaths: ["page.properties.testMap.nestedMap", "page.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["page.properties.testMap.nestedMap", "page.context.ctestMap.cnestedMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -392,5 +397,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PAGES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js index 3eec47b89d..68b5aafcb5 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js @@ -55,6 +55,11 @@ module.exports = { jsonPaths: ["screen.properties.testMap.nestedMap", "screen.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["screen.properties.testMap.nestedMap", "screen.context.ctestMap.cnestedMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -392,5 +397,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "SCREENS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js index 6b411835db..70c20b8b8e 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js @@ -78,6 +78,15 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "track.userProperties.UPMap.nestedMap", + "track.properties.PMap.nestedMap", + "track.context.CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -828,6 +837,148 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "TRACKS" + } + }, + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EMAIL": "test@gmail.com", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EMAIL": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/names.js b/test/__tests__/data/warehouse/names.js index b0380dc06d..e6015cef2c 100644 --- a/test/__tests__/data/warehouse/names.js +++ b/test/__tests__/data/warehouse/names.js @@ -41,6 +41,19 @@ const names = { C_Z: "string", CAMEL_CASE_123_KEY: "string", _1_C_COMEGA: "string" + }, + snowpipe_streaming: { + OMEGA: "string", + OMEGA_V_2: "string", + _9_MEGA: "string", + MEGA: "string", + OME_GA: "string", + ALPHA: "string", + OME_GA: "string", + _9_MEGA_90: "string", + C_Z: "string", + CAMEL_CASE_123_KEY: "string", + _1_C_COMEGA: "string" } }, data: { @@ -69,6 +82,19 @@ const names = { C_Z: "test", CAMEL_CASE_123_KEY: "test", _1_C_COMEGA: "test" + }, + snowpipe_streaming: { + OMEGA: "test", + OMEGA_V_2: "test", + _9_MEGA: "test", + MEGA: "test", + OME_GA: "test", + ALPHA: "test", + OME_GA: "test", + _9_MEGA_90: "test", + C_Z: "test", + CAMEL_CASE_123_KEY: "test", + _1_C_COMEGA: "test" } }, namesMap: { @@ -97,6 +123,19 @@ const names = { Cízǔ: "C_Z", CamelCase123Key: "CAMEL_CASE_123_KEY", "1CComega": "_1_C_COMEGA" + }, + snowpipe_streaming: { + omega: "OMEGA", + "omega v2": "OMEGA_V_2", + "9mega": "_9_MEGA", + "mega&": "MEGA", + ome$ga: "OME_GA", + alpha$: "ALPHA", + "ome_ ga": "OME_GA", + "9mega________-________90": "_9_MEGA_90", + Cízǔ: "C_Z", + CamelCase123Key: "CAMEL_CASE_123_KEY", + "1CComega": "_1_C_COMEGA" } } } diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 7fdecbd7cf..44d1d33ff8 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -37,14 +37,6 @@ const integrations = [ "s3_datalake", "gcs_datalake", ]; - -const integration = (index ) => { - const it = integrations[index]; - if (it === "snowflake" || it === "snowpipe_streaming") { - return "snowflake"; - } - return it; -} const transformers = integrations.map(integration => require(`../../src/${version}/destinations/${integration}/transform`) ); @@ -62,7 +54,7 @@ const propsKeyMap = { }; const integrationCasedString = (integration, str) => { - if (integration === "snowflake") { + if (integration === "snowflake" || integration === "snowpipe_streaming") { return str.toUpperCase(); } return str; @@ -74,7 +66,7 @@ describe("event types", () => { const i = input("track"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("track", integration(index))); + expect(received).toMatchObject(output("track", integrations[index])); }); }); }); @@ -85,7 +77,7 @@ describe("event types", () => { // also verfies priority order between traits and context.traits transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("identify", integration(index))); + expect(received).toMatchObject(output("identify", integrations[index])); }); }); }); @@ -95,7 +87,7 @@ describe("event types", () => { const i = input("page"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("page", integration(index))); + expect(received).toMatchObject(output("page", integrations[index])); }); }); it("should take name from properties if top-level name is missing", () => { @@ -104,7 +96,7 @@ describe("event types", () => { delete i.message.name; transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("page", integration(index))); + expect(received).toMatchObject(output("page", integrations[index])); }); }); }); @@ -114,7 +106,7 @@ describe("event types", () => { const i = input("screen"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integration(index))); + expect(received).toMatchObject(output("screen", integrations[index])); }); }); it("should take name from properties if top-level name is missing", () => { @@ -123,7 +115,7 @@ describe("event types", () => { delete i.message.name; transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integration(index))); + expect(received).toMatchObject(output("screen", integrations[index])); }); }); }); @@ -133,7 +125,7 @@ describe("event types", () => { const i = input("alias"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("alias", integration(index))); + expect(received).toMatchObject(output("alias", integrations[index])); }); }); }); @@ -143,7 +135,7 @@ describe("event types", () => { const i = input("extract"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("extract", integration(index))); + expect(received).toMatchObject(output("extract", integrations[index])); }); }); }); @@ -161,8 +153,9 @@ describe("column & table names", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); - const provider = - (integration(index) === "snowflake" || integration(index) == "snowpipe_streaming") ? "snowflake" : "default"; + const provider = ["snowflake", "snowpipe_streaming"].includes(integrations[index]) + ? integrations[index] + : "default"; expect(received[1].metadata.columns).toMatchObject( names.output.columns[provider] @@ -195,7 +188,7 @@ describe("column & table names", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); - if (integration(index) === "postgres") { + if (integrations[index] === "postgres") { expect(received[1].metadata).toHaveProperty( "table", "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1" @@ -219,7 +212,7 @@ describe("column & table names", () => { //KEY should be trimmed to 63 return; } - if (integration(index) === "snowflake") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { expect(received[1].metadata).toHaveProperty( "table", "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2" @@ -242,7 +235,7 @@ describe("column & table names", () => { ); return; } - if (integration(index) === "s3_datalake" || integration(index) === "gcs_datalake" || integration(index) === "azure_datalake") { + if (integrations[index] === "s3_datalake" || integrations[index] === "gcs_datalake" || integrations[index] === "azure_datalake") { expect(received[1].metadata).toHaveProperty( "table", "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5" @@ -325,7 +318,7 @@ describe("conflict between rudder set props and user set props", () => { const propsKey = propsKeyMap[evType]; transformers.forEach((transformer, index) => { let sampleRudderPropKey = "id"; - if (integration(index) === "snowflake") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { sampleRudderPropKey = "ID"; } @@ -363,7 +356,7 @@ describe("handle reserved words", () => { const propsKey = propsKeyMap[evType]; transformers.forEach((transformer, index) => { const reserverdKeywordsMap = - reservedANSIKeywordsMap[integration(index).toUpperCase()]; + reservedANSIKeywordsMap[integrations[index].toUpperCase()]; i.message[propsKey] = Object.assign( i.message[propsKey] || {}, @@ -373,7 +366,7 @@ describe("handle reserved words", () => { const received = transformer.process(i); const out = - evType === "track" || evType === "identify" + evType === "track" || (evType === "identify" && integrations[index] !== 'snowpipe_streaming') ? received[1] : received[0]; @@ -386,7 +379,7 @@ describe("handle reserved words", () => { } else { k = snakeCasedKey; } - if (integration(index) === "snowflake") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { expect(out.metadata.columns).toHaveProperty(k); } else { expect(out.metadata.columns).toHaveProperty(k.toLowerCase()); @@ -470,24 +463,24 @@ describe("context ip", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[0].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("new_ip"); if (received[1]) { expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[1].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("new_ip"); } @@ -505,23 +498,23 @@ describe("context ip", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[0].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("requested_ip"); if (received[1]) { expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[1].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("requested_ip"); } @@ -542,10 +535,10 @@ describe("remove rudder property if rudder property is null", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ); }); }); @@ -560,29 +553,29 @@ describe("remove any property if event is object ", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); }); transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "event_text") + integrationCasedString(integrations[index], "event_text") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "event_text") + integrationCasedString(integrations[index], "event_text") ); }); i.message.channel = { channel: "android" }; transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); }); }); @@ -598,13 +591,13 @@ describe("store full rudder event", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); const columnName = integrationCasedString( - integration(index), + integrations[index], "rudder_event" ); expect(received[0].metadata.columns).toHaveProperty(columnName); expect(received[0].metadata.columns[columnName]).toEqual( - fullEventColumnTypeByProvider[integration(index)] + fullEventColumnTypeByProvider[integrations[index]] ); expect(received[0].data[columnName]).toEqual(JSON.stringify(i.message)); @@ -644,7 +637,7 @@ describe("rudder reserved columns", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); checkProps.forEach(k => { - k = integrationCasedString(integration(index), k); + k = integrationCasedString(integrations[index], k); expect(received[0].metadata.columns).not.toHaveProperty(k); expect(received[0].data).not.toHaveProperty(k); if (received[1]) { @@ -664,14 +657,25 @@ describe("id column datatype for users table", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); + if (integrations[index] === 'snowpipe_streaming') { + expect(received).toHaveLength(1); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "user_id") + ] + ).toEqual("int"); + return; + } + + expect(received).toHaveLength(2); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "user_id") + integrationCasedString(integrations[index], "user_id") ] ).toEqual("int"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); }); @@ -681,14 +685,25 @@ describe("id column datatype for users table", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); + if (integrations[index] === 'snowpipe_streaming') { + expect(received).toHaveLength(1); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "user_id") + ] + ).toEqual("float"); + return; + } + + expect(received).toHaveLength(2); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "user_id") + integrationCasedString(integrations[index], "user_id") ] ).toEqual("float"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("float"); }); @@ -707,22 +722,22 @@ describe("handle leading underscores in properties", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "_timestamp") + integrationCasedString(integrations[index], "_timestamp") ); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "__timestamp") + integrationCasedString(integrations[index], "__timestamp") ); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "__timestamp_new") + integrationCasedString(integrations[index], "__timestamp_new") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "_timestamp") + integrationCasedString(integrations[index], "_timestamp") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "__timestamp") + integrationCasedString(integrations[index], "__timestamp") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "__timestamp_new") + integrationCasedString(integrations[index], "__timestamp_new") ); }); }); @@ -736,22 +751,22 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); }); }); @@ -764,22 +779,22 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); }); }); @@ -792,17 +807,17 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("string"); }); @@ -818,28 +833,28 @@ describe("handle recordId from cloud sources", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toEqual("string"); expect( received[0].data[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toBe("42"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toBe(42); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); }); }); @@ -854,28 +869,28 @@ describe("handle recordId from cloud sources", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toEqual("string"); expect( received[0].data[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toBe("42"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toBe(42); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); }); }); @@ -915,10 +930,10 @@ describe("handle level three nested events from sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); }); }); @@ -943,10 +958,10 @@ describe("handle level three nested events from sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); }); }); @@ -956,7 +971,7 @@ describe("Handle no of columns in an event", () => { it("should throw an error if no of columns are more than 200", () => { const i = input("track"); transformers - .filter((transformer, index) => integration(index) !== "s3_datalake" && integration(index) !== "gcs_datalake" && integration(index) !== "azure_datalake") + .filter((transformer, index) => integrations[index] !== "s3_datalake" && integrations[index] !== "gcs_datalake" && integrations[index] !== "azure_datalake") .forEach((transformer, index) => { i.message.properties = largeNoOfColumnsevent; expect(() => transformer.process(i)).toThrow( @@ -984,13 +999,13 @@ describe("Add auto generated messageId for events missing it", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ); expect(received[0].data).toHaveProperty( - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toMatch(/auto-.*/); }); }); @@ -1007,10 +1022,10 @@ describe("Add receivedAt for events missing it", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "received_at") + integrationCasedString(integrations[index], "received_at") ); expect(received[0].data).toHaveProperty( - integrationCasedString(integration(index), "received_at") + integrationCasedString(integrations[index], "received_at") ); }); }); @@ -1023,22 +1038,22 @@ describe("Integration options", () => { const i = opInput("track"); transformers.forEach((transformer, index) => { const {jsonPaths} = i.destination.Config; - if (integration(index) === "postgres") { + if (integrations[index] === "postgres") { delete i.destination.Config.jsonPaths; } const received = transformer.process(i); i.destination.Config.jsonPaths = jsonPaths; - expect(received).toEqual(opOutput("track", integration(index))); + expect(received).toEqual(opOutput("track", integrations[index])); }); }); }); describe("users", () => { it("should skip users when skipUsersTable is set", () => { - const i = opInput("users"); + const i = opInput("identify"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toEqual(opOutput("users", integration(index))); + expect(received).toEqual(opOutput("identify", integrations[index])); }); }); }); @@ -1054,6 +1069,8 @@ describe("Integration options", () => { return _.cloneDeep(config.output.postgres); case "snowflake": return _.cloneDeep(config.output.snowflake); + case "snowpipe_streaming": + return _.cloneDeep(config.output.snowpipe_streaming); case "s3_datalake": case "gcs_datalake": case "azure_datalake": @@ -1093,18 +1110,18 @@ describe("Integration options", () => { for (const testCase of testCases) { transformers.forEach((transformer, index) => { - it(`new ${testCase.eventType} for ${integration(index)}`, () => { + it(`new ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(testCase.eventType, config, integration(index))); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) - it(`legacy ${testCase.eventType} for ${integration(index)}`, () => { + it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(testCase.eventType, config, integration(index))); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) }); } @@ -1242,7 +1259,10 @@ describe("Destination config", () => { transformers.forEach((transformer, index) => { const received = transformer.process(scenario.event); - expect(received).toHaveLength(scenario.expected.length); + const expectedLength = integrations[index] === "snowpipe_streaming" && scenario.event.message.type === "identify" + ? 1 + : scenario.expected.length; + expect(received).toHaveLength(expectedLength); for (const i in received) { const evt = received[i]; expect(evt.data.id ? evt.data.id : evt.data.ID).toEqual(scenario.expected[i].id); @@ -1285,7 +1305,9 @@ describe("Destination config", () => { } } const output = transformer.process(event); - const events = [output[0], output[1]]; // identifies and users event + const events = integrations[index] === "snowpipe_streaming" + ? [output[0]] + : [output[0], output[1]]; const traitsToCheck = { 'city': 'Disney', 'country': 'USA', @@ -1294,10 +1316,10 @@ describe("Destination config", () => { }; events.forEach(event => { Object.entries(traitsToCheck).forEach(([trait, value]) => { - expect(event.data[integrationCasedString(integration(index), trait)]).toEqual(value); - expect(event.data[integrationCasedString(integration(index), `context_traits_${trait}`)]).toEqual(value); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_traits_${trait}`)); + expect(event.data[integrationCasedString(integrations[index], trait)]).toEqual(value); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); }); }); }); @@ -1325,13 +1347,13 @@ describe("Destination config", () => { } } const output = transformer.process(event); - expect(output[0].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v_2'); - expect(output[0].data[integrationCasedString(integration(index), `context_attribute_v_3`)]).toEqual('some-value'); - expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v_3`)); - expect(output[1].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v_2'); - expect(output[1].data[integrationCasedString(integration(index), `context_attribute_v_3`)]).toEqual('some-value'); - expect(output[1].metadata.table).toEqual(integrationCasedString(integration(index), 'button_clicked_v_2')); - expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v_3`)); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v_2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); }); }); }); @@ -1364,7 +1386,10 @@ describe("Destination config", () => { } } const received = transformer.process(event); - const events = [received[0], received[1]]; // identifies and users event + const events = integrations[index] === "snowpipe_streaming" + ? [received[0]] + : [received[0], received[1]]; + // identifies and users event const traitsToCheck = { 'city': 'Disney', 'country': 'USA', @@ -1373,10 +1398,10 @@ describe("Destination config", () => { }; events.forEach(event => { Object.entries(traitsToCheck).forEach(([trait, value]) => { - expect(event.data).not.toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.data[integrationCasedString(integration(index), `context_traits_${trait}`)]).toEqual(value); - expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_traits_${trait}`)); + expect(event.data).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); }); }); }); @@ -1402,13 +1427,13 @@ describe("Destination config", () => { } } const output = transformer.process(event); - expect(output[0].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v2'); - expect(output[0].data[integrationCasedString(integration(index), `context_attribute_v3`)]).toEqual('some-value'); - expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v3`)); - expect(output[1].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v2'); - expect(output[1].data[integrationCasedString(integration(index), `context_attribute_v3`)]).toEqual('some-value'); - expect(output[1].metadata.table).toEqual(integrationCasedString(integration(index), 'button_clicked_v2')); - expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v3`)); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); }); }); }); @@ -1615,8 +1640,8 @@ describe("context traits", () => { expect(Object.keys(received[0].data).join()).not.toMatch(/context_traits/g); } for (const column of t.expectedColumns) { - expect(received[0].metadata.columns[integrationCasedString(integration(index), column)]).toEqual(t.expectedMetadata); - expect(received[0].data[integrationCasedString(integration(index), column)]).toEqual(t.expectedData); + expect(received[0].metadata.columns[integrationCasedString(integrations[index], column)]).toEqual(t.expectedMetadata); + expect(received[0].data[integrationCasedString(integrations[index], column)]).toEqual(t.expectedData); } }); } @@ -1708,8 +1733,8 @@ describe("group traits", () => { expect(Object.keys(received[0].data).join()).not.toMatch(/group_traits/g); } for (const column of t.expectedColumns) { - expect(received[0].metadata.columns[integrationCasedString(integration(index), column)]).toEqual(t.expectedMetadata); - expect(received[0].data[integrationCasedString(integration(index), column)]).toEqual(t.expectedData); + expect(received[0].metadata.columns[integrationCasedString(integrations[index], column)]).toEqual(t.expectedMetadata); + expect(received[0].data[integrationCasedString(integrations[index], column)]).toEqual(t.expectedData); } }); }); From ebcf84e07bf121d882c99df973af265a915a1ce1 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 9 Dec 2024 13:10:01 +0530 Subject: [PATCH 4/4] fix: braze subscription group fixes (#3901) --- src/util/prometheus.js | 12 ++ src/v0/destinations/braze/braze.util.test.js | 151 ++++++++++++++++-- src/v0/destinations/braze/config.js | 2 +- src/v0/destinations/braze/transform.js | 2 +- src/v0/destinations/braze/util.js | 54 ++++++- .../destinations/braze/processor/data.ts | 4 +- .../destinations/braze/router/data.ts | 33 +++- 7 files changed, 240 insertions(+), 18 deletions(-) diff --git a/src/util/prometheus.js b/src/util/prometheus.js index a4c12e4ea8..2a3a1fb22a 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -520,6 +520,18 @@ class Prometheus { type: 'counter', labelNames: ['destination_id'], }, + { + name: 'braze_batch_subscription_size', + help: 'braze_batch_subscription_size', + type: 'gauge', + labelNames: ['destination_id'], + }, + { + name: 'braze_batch_subscription_combined_size', + help: 'braze_batch_subscription_combined_size', + type: 'gauge', + labelNames: ['destination_id'], + }, { name: 'mailjet_packing_size', help: 'mailjet_packing_size', diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js index 985f2434d5..7d10035233 100644 --- a/src/v0/destinations/braze/braze.util.test.js +++ b/src/v0/destinations/braze/braze.util.test.js @@ -6,6 +6,7 @@ const { getPurchaseObjs, setAliasObject, handleReservedProperties, + combineSubscriptionGroups, } = require('./util'); const { processBatch } = require('./util'); const { @@ -958,7 +959,9 @@ describe('processBatch', () => { attributes: [{ id: i, name: 'test', xyz: 'abc' }], events: [{ id: i, event: 'test', xyz: 'abc' }], purchases: [{ id: i, purchase: 'test', xyz: 'abc' }], - subscription_groups: [{ id: i, group: 'test', xyz: 'abc' }], + subscription_groups: [ + { subscription_group_id: i, group: 'test', subscription_state: 'abc' }, + ], merge_updates: [{ id: i, alias: 'test', xyz: 'abc' }], }, }, @@ -972,7 +975,7 @@ describe('processBatch', () => { // Assert that the response is as expected expect(result.length).toBe(1); // One successful batched request and one failure response - expect(result[0].batchedRequest.length).toBe(6); // Two batched requests + expect(result[0].batchedRequest.length).toBe(8); // Two batched requests expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events @@ -981,10 +984,12 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.attributes.length).toBe(25); // Second batch contains remaining 25 attributes expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(25); // Second batch contains remaining 25 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(25); // Second batch contains remaining 25 purchases - expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); - expect(result[0].batchedRequest[4].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates - expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates + expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group + expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Third batch contains 25 subscription group + expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(25); // Fourth batch contains 25 subscription group + expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates + expect(result[0].batchedRequest[7].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates }); test('processBatch handles more than 75 attributes, events, and purchases with non uniform distribution', () => { @@ -1055,7 +1060,9 @@ describe('processBatch', () => { batchedRequest: { body: { JSON: { - subscription_groups: [{ id: i, group: 'test', xyz: 'abc' }], + subscription_groups: [ + { subscription_group_id: i, group: 'test', subscription_state: 'abc' }, + ], }, }, }, @@ -1093,7 +1100,7 @@ describe('processBatch', () => { // Assert that the response is as expected expect(result.length).toBe(1); // One successful batched request and one failure response expect(result[0].metadata.length).toBe(490); // Check the total length is same as input jobs (120 + 160 + 100 + 70 +40) - expect(result[0].batchedRequest.length).toBe(6); // Two batched requests + expect(result[0].batchedRequest.length).toBe(7); // Two batched requests expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events @@ -1103,9 +1110,10 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(45); // Second batch contains remaining 45 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(75); // Second batch contains remaining 75 purchases expect(result[0].batchedRequest[2].body.JSON.purchases.length).toBe(10); // Third batch contains remaining 10 purchases - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group - expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(20); // Second batch contains 20 subscription group - expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(40); // First batch contains 40 merge_updates + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group + expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group + expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(20); // Third batch contains 20 subscription group + expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(40); // First batch contains 50 merge_updates }); test('check success and failure scenarios both for processBatch', () => { @@ -1751,3 +1759,124 @@ describe('handleReservedProperties', () => { }); }); }); + +describe('combineSubscriptionGroups', () => { + it('should merge external_ids, emails, and phones for the same subscription_group_id and subscription_state', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1', 'id2'], + emails: ['email1@example.com', 'email2@example.com'], + phones: ['+1234567890', '+0987654321'], + }, + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id2', 'id3'], + emails: ['email2@example.com', 'email3@example.com'], + phones: ['+1234567890', '+1122334455'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1', 'id2', 'id3'], + emails: ['email1@example.com', 'email2@example.com', 'email3@example.com'], + phones: ['+1234567890', '+0987654321', '+1122334455'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); + + it('should handle groups with missing external_ids, emails, or phones', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + emails: ['email1@example.com'], + }, + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + phones: ['+1234567890'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + emails: ['email1@example.com'], + phones: ['+1234567890'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); + + it('should handle multiple unique subscription groups', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + { + subscription_group_id: 'group2', + subscription_state: 'Unsubscribed', + external_ids: ['id2'], + emails: ['email2@example.com'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + { + subscription_group_id: 'group2', + subscription_state: 'Unsubscribed', + external_ids: ['id2'], + emails: ['email2@example.com'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); + + it('should not include undefined fields in the output', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); +}); diff --git a/src/v0/destinations/braze/config.js b/src/v0/destinations/braze/config.js index 2bbade2754..8ccabbdb0a 100644 --- a/src/v0/destinations/braze/config.js +++ b/src/v0/destinations/braze/config.js @@ -36,7 +36,7 @@ const IDENTIFY_BRAZE_MAX_REQ_COUNT = 50; // https://www.braze.com/docs/api/endpoints/user_data/post_user_delete/ const ALIAS_BRAZE_MAX_REQ_COUNT = 50; -const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 50; +const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 25; const DEL_MAX_BATCH_SIZE = 50; const DESTINATION = 'braze'; diff --git a/src/v0/destinations/braze/transform.js b/src/v0/destinations/braze/transform.js index 09fb0205c5..2a9cb8e7dc 100644 --- a/src/v0/destinations/braze/transform.js +++ b/src/v0/destinations/braze/transform.js @@ -383,7 +383,7 @@ function processGroup(message, destination) { ); } subscriptionGroup.subscription_state = message.traits.subscriptionState; - subscriptionGroup.external_id = [message.userId]; + subscriptionGroup.external_ids = [message.userId]; const phone = getFieldValueFromMessage(message, 'phone'); const email = getFieldValueFromMessage(message, 'email'); if (phone) { diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js index 6c8cf64265..3778c34c43 100644 --- a/src/v0/destinations/braze/util.js +++ b/src/v0/destinations/braze/util.js @@ -45,6 +45,42 @@ const getEndpointFromConfig = (destination) => { return endpoint; }; +// Merges external_ids, emails, and phones for entries with the same subscription_group_id and subscription_state +const combineSubscriptionGroups = (subscriptionGroups) => { + const uniqueGroups = {}; + + subscriptionGroups.forEach((group) => { + const key = `${group.subscription_group_id}-${group.subscription_state}`; + if (!uniqueGroups[key]) { + uniqueGroups[key] = { + ...group, + external_ids: [...(group.external_ids || [])], + emails: [...(group.emails || [])], + phones: [...(group.phones || [])], + }; + } else { + uniqueGroups[key].external_ids.push(...(group.external_ids || [])); + uniqueGroups[key].emails.push(...(group.emails || [])); + uniqueGroups[key].phones.push(...(group.phones || [])); + } + }); + + return Object.values(uniqueGroups).map((group) => { + const result = { + subscription_group_id: group.subscription_group_id, + subscription_state: group.subscription_state, + external_ids: [...new Set(group.external_ids)], + }; + if (group.emails?.length) { + result.emails = [...new Set(group.emails)]; + } + if (group.phones?.length) { + result.phones = [...new Set(group.phones)]; + } + return result; + }); +}; + const CustomAttributeOperationUtil = { customAttributeUpdateOperation(key, data, traits, mergeObjectsUpdateOperation) { data[key] = {}; @@ -381,8 +417,22 @@ function prepareGroupAndAliasBatch(arrayChunks, responseArray, destination, type } else if (type === 'subscription') { response.endpoint = getSubscriptionGroupEndPoint(getEndpointFromConfig(destination)); const subscription_groups = chunk; + // maketool transformed event + logger.info(`braze subscription chunk ${JSON.stringify(subscription_groups)}`); + + stats.gauge('braze_batch_subscription_size', subscription_groups.length, { + destination_id: destination.ID, + }); + + // Deduplicate the subscription groups before constructing the response body + const deduplicatedSubscriptionGroups = combineSubscriptionGroups(subscription_groups); + + stats.gauge('braze_batch_subscription_combined_size', deduplicatedSubscriptionGroups.length, { + destination_id: destination.ID, + }); + response.body.JSON = removeUndefinedAndNullValues({ - subscription_groups, + subscription_groups: deduplicatedSubscriptionGroups, }); } responseArray.push({ @@ -490,6 +540,7 @@ const processBatch = (transformedEvents) => { prepareGroupAndAliasBatch(mergeUsersArrayChunks, responseArray, destination, 'merge'); if (successMetadata.length > 0) { + console.log(`Response 1 batchRequest ${JSON.stringify(responseArray)}`); finalResponse.push({ batchedRequest: responseArray, metadata: successMetadata, @@ -756,4 +807,5 @@ module.exports = { collectStatsForAliasFailure, collectStatsForAliasMissConfigurations, handleReservedProperties, + combineSubscriptionGroups, }; diff --git a/test/integrations/destinations/braze/processor/data.ts b/test/integrations/destinations/braze/processor/data.ts index 240206791e..9b6f6dac65 100644 --- a/test/integrations/destinations/braze/processor/data.ts +++ b/test/integrations/destinations/braze/processor/data.ts @@ -3160,7 +3160,7 @@ export const data = [ { subscription_group_id: '1234', subscription_state: 'subscribed', - external_id: ['Randomuser2222'], + external_ids: ['Randomuser2222'], phones: ['5055077683'], }, ], @@ -3281,7 +3281,7 @@ export const data = [ { subscription_group_id: '1234', subscription_state: 'unsubscribed', - external_id: ['Randomuser2222'], + external_ids: ['Randomuser2222'], emails: ['abc@test.com'], }, ], diff --git a/test/integrations/destinations/braze/router/data.ts b/test/integrations/destinations/braze/router/data.ts index 6803742e86..b788e22741 100644 --- a/test/integrations/destinations/braze/router/data.ts +++ b/test/integrations/destinations/braze/router/data.ts @@ -219,6 +219,34 @@ export const data = [ metadata: { jobId: 6, userId: 'u1' }, message: { type: 'alias', previousId: 'adsfsaf2', userId: 'dsafsdf2' }, }, + { + destination: { + Config: { + restApiKey: 'dummyApiKey', + prefixProperties: true, + useNativeSDK: false, + dataCenter: 'us-01', + enableSubscriptionGroupInGroupCall: true, + }, + DestinationDefinition: { + DisplayName: 'Braze', + ID: '1WhbSZ6uA3H5ChVifHpfL2H6sie', + Name: 'BRAZE', + }, + Enabled: true, + ID: '1WhcOCGgj9asZu850HvugU2C3Aq', + Name: 'Braze', + Transformations: [], + }, + metadata: { jobId: 7, userId: 'u1' }, + message: { + anonymousId: '56yrtsdfgbgxcb-22b4-401d-aae5-1b994be9afdf', + groupId: 'c90f0fd2-2a02-4f2f-bf07-7e7d2c2ed2b1', + traits: { phone: '5055077683', subscriptionState: 'subscribed' }, + userId: 'user12345', + type: 'group', + }, + }, ], destType: 'braze', }, @@ -299,13 +327,13 @@ export const data = [ JSON: { subscription_groups: [ { - external_id: ['user123'], + external_ids: ['user123', 'user12345'], phones: ['5055077683'], subscription_group_id: 'c90f0fd2-2a02-4f2f-bf07-7e7d2c2ed2b1', subscription_state: 'subscribed', }, { - external_id: ['user877'], + external_ids: ['user877'], phones: ['5055077683'], subscription_group_id: '58d0a278-b55b-4f10-b7d2-98d1c5dd4c30', subscription_state: 'subscribed', @@ -356,6 +384,7 @@ export const data = [ { jobId: 4, userId: 'u1' }, { jobId: 5, userId: 'u1' }, { jobId: 6, userId: 'u1' }, + { jobId: 7, userId: 'u1' }, ], batched: true, statusCode: 200,