diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index c7e59dce86..ba62e5d8d1 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -96,15 +96,15 @@ jobs: with: ref: ${{ needs.get_sha.outputs.sha }} fetch-depth: 1 - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3.7.1 - + - name: Login to DockerHub uses: docker/login-action@v3.3.0 with: username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} + password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3.7.1 - name: Build Docker Image uses: docker/build-push-action@v6.9.0 @@ -149,16 +149,16 @@ jobs: with: ref: ${{ needs.get_sha.outputs.sha }} fetch-depth: 1 - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3.7.1 - + - name: Login to DockerHub uses: docker/login-action@v3.3.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3.7.1 + - name: Build Docker Image uses: docker/build-push-action@v6.9.0 with: @@ -198,15 +198,15 @@ jobs: needs: [build-transformer-image-amd64, build-transformer-image-arm64] steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.7.1 - - name: Login to DockerHub uses: docker/login-action@v3.3.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.7.1 + - name: Create multi-arch manifest run: | docker buildx imagetools create -t ${{ inputs.push_tags }} ${{ inputs.push_tags }}-amd64 ${{ inputs.push_tags }}-arm64 diff --git a/.github/workflows/dt-test-and-report-code-coverage.yml b/.github/workflows/dt-test-and-report-code-coverage.yml index 2c76898882..12bd9ac78b 100644 --- a/.github/workflows/dt-test-and-report-code-coverage.yml +++ b/.github/workflows/dt-test-and-report-code-coverage.yml @@ -33,7 +33,7 @@ jobs: - name: Checkout uses: actions/checkout@v4.2.1 with: - fetch-depth: 1 + fetch-depth: 0 - name: Setup Node uses: actions/setup-node@v4.0.4 diff --git a/.github/workflows/prepare-for-staging-deploy.yml b/.github/workflows/prepare-for-staging-deploy.yml index 8eeafc376e..cdf27f7832 100644 --- a/.github/workflows/prepare-for-staging-deploy.yml +++ b/.github/workflows/prepare-for-staging-deploy.yml @@ -78,6 +78,7 @@ jobs: runs-on: ubuntu-latest needs: [generate-tag-names, build-transformer-image, build-user-transformer-image] env: + TF_IMAGE_REPOSITORY: rudderstack/rudder-transformer TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name }} UT_TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name_ut }} steps: @@ -113,6 +114,7 @@ jobs: git checkout -b $BRANCH_NAME cd helm-charts/shared-services/per-az/environment/staging + yq eval -i ".rudder-transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" staging.yaml yq eval -i ".rudder-transformer.image.tag=\"$TAG_NAME\"" staging.yaml yq eval -i ".user-transformer.image.tag=\"$TAG_NAME\"" staging.yaml git add staging.yaml diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4bd66285bd..604ff1eaee 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -35,8 +35,8 @@ jobs: - name: Filter JS/TS Files run: | - echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' > changed_files.txt - if [ ! -s changed_files.txt ]; then + changed_files=$(echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' || true) + if [ -z "$changed_files" ]; then echo "No JS/TS files to format or lint." exit 0 fi diff --git a/CHANGELOG.md b/CHANGELOG.md index bb1779fd3a..e742ea1f55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.86.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.1...v1.86.0) (2024-12-09) + + +### Features + +* add support for getting anonymousId by note attributes array ([53d2bef](https://github.com/rudderlabs/rudder-transformer/commit/53d2bef38ad5d7bc5504111ec797b3c3973546dd)) +* add support for getting anonymousId by note attributes array ([#3893](https://github.com/rudderlabs/rudder-transformer/issues/3893)) ([d7f390c](https://github.com/rudderlabs/rudder-transformer/commit/d7f390cb471e44afb276484b8b804d1f257c539c)) + + +### Bug Fixes + +* braze subscription group fixes ([#3901](https://github.com/rudderlabs/rudder-transformer/issues/3901)) ([ebcf84e](https://github.com/rudderlabs/rudder-transformer/commit/ebcf84e07bf121d882c99df973af265a915a1ce1)) +* remove redundant ids and userIdentifier when gbraid or wbraid are there ([#3910](https://github.com/rudderlabs/rudder-transformer/issues/3910)) ([313710c](https://github.com/rudderlabs/rudder-transformer/commit/313710ca725538e5ffe357216d9c88e444f995c8)) +* skipping users events for snowpipe streaming ([#3836](https://github.com/rudderlabs/rudder-transformer/issues/3836)) ([12621c8](https://github.com/rudderlabs/rudder-transformer/commit/12621c8eee641f5a03a997e95ed016cff0eefde7)) + ### [1.85.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.0...v1.85.1) (2024-11-21) diff --git a/package-lock.json b/package-lock.json index 40033e278b..991d341e9f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.85.1", + "version": "1.86.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.85.1", + "version": "1.86.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index e0b17a4f47..407ee1d1ab 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.85.1", + "version": "1.86.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { diff --git a/src/util/prometheus.js b/src/util/prometheus.js index c8dd55068b..2a3a1fb22a 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -520,6 +520,24 @@ class Prometheus { type: 'counter', labelNames: ['destination_id'], }, + { + name: 'braze_batch_subscription_size', + help: 'braze_batch_subscription_size', + type: 'gauge', + labelNames: ['destination_id'], + }, + { + name: 'braze_batch_subscription_combined_size', + help: 'braze_batch_subscription_combined_size', + type: 'gauge', + labelNames: ['destination_id'], + }, + { + name: 'mailjet_packing_size', + help: 'mailjet_packing_size', + type: 'gauge', + labelNames: ['group'], + }, { name: 'hs_batch_size', help: 'hs_batch_size', diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js index 985f2434d5..7d10035233 100644 --- a/src/v0/destinations/braze/braze.util.test.js +++ b/src/v0/destinations/braze/braze.util.test.js @@ -6,6 +6,7 @@ const { getPurchaseObjs, setAliasObject, handleReservedProperties, + combineSubscriptionGroups, } = require('./util'); const { processBatch } = require('./util'); const { @@ -958,7 +959,9 @@ describe('processBatch', () => { attributes: [{ id: i, name: 'test', xyz: 'abc' }], events: [{ id: i, event: 'test', xyz: 'abc' }], purchases: [{ id: i, purchase: 'test', xyz: 'abc' }], - subscription_groups: [{ id: i, group: 'test', xyz: 'abc' }], + subscription_groups: [ + { subscription_group_id: i, group: 'test', subscription_state: 'abc' }, + ], merge_updates: [{ id: i, alias: 'test', xyz: 'abc' }], }, }, @@ -972,7 +975,7 @@ describe('processBatch', () => { // Assert that the response is as expected expect(result.length).toBe(1); // One successful batched request and one failure response - expect(result[0].batchedRequest.length).toBe(6); // Two batched requests + expect(result[0].batchedRequest.length).toBe(8); // Two batched requests expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events @@ -981,10 +984,12 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.attributes.length).toBe(25); // Second batch contains remaining 25 attributes expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(25); // Second batch contains remaining 25 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(25); // Second batch contains remaining 25 purchases - expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); - expect(result[0].batchedRequest[4].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates - expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates + expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group + expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Third batch contains 25 subscription group + expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(25); // Fourth batch contains 25 subscription group + expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates + expect(result[0].batchedRequest[7].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates }); test('processBatch handles more than 75 attributes, events, and purchases with non uniform distribution', () => { @@ -1055,7 +1060,9 @@ describe('processBatch', () => { batchedRequest: { body: { JSON: { - subscription_groups: [{ id: i, group: 'test', xyz: 'abc' }], + subscription_groups: [ + { subscription_group_id: i, group: 'test', subscription_state: 'abc' }, + ], }, }, }, @@ -1093,7 +1100,7 @@ describe('processBatch', () => { // Assert that the response is as expected expect(result.length).toBe(1); // One successful batched request and one failure response expect(result[0].metadata.length).toBe(490); // Check the total length is same as input jobs (120 + 160 + 100 + 70 +40) - expect(result[0].batchedRequest.length).toBe(6); // Two batched requests + expect(result[0].batchedRequest.length).toBe(7); // Two batched requests expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events @@ -1103,9 +1110,10 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(45); // Second batch contains remaining 45 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(75); // Second batch contains remaining 75 purchases expect(result[0].batchedRequest[2].body.JSON.purchases.length).toBe(10); // Third batch contains remaining 10 purchases - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group - expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(20); // Second batch contains 20 subscription group - expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(40); // First batch contains 40 merge_updates + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group + expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group + expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(20); // Third batch contains 20 subscription group + expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(40); // First batch contains 50 merge_updates }); test('check success and failure scenarios both for processBatch', () => { @@ -1751,3 +1759,124 @@ describe('handleReservedProperties', () => { }); }); }); + +describe('combineSubscriptionGroups', () => { + it('should merge external_ids, emails, and phones for the same subscription_group_id and subscription_state', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1', 'id2'], + emails: ['email1@example.com', 'email2@example.com'], + phones: ['+1234567890', '+0987654321'], + }, + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id2', 'id3'], + emails: ['email2@example.com', 'email3@example.com'], + phones: ['+1234567890', '+1122334455'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1', 'id2', 'id3'], + emails: ['email1@example.com', 'email2@example.com', 'email3@example.com'], + phones: ['+1234567890', '+0987654321', '+1122334455'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); + + it('should handle groups with missing external_ids, emails, or phones', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + emails: ['email1@example.com'], + }, + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + phones: ['+1234567890'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + emails: ['email1@example.com'], + phones: ['+1234567890'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); + + it('should handle multiple unique subscription groups', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + { + subscription_group_id: 'group2', + subscription_state: 'Unsubscribed', + external_ids: ['id2'], + emails: ['email2@example.com'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + { + subscription_group_id: 'group2', + subscription_state: 'Unsubscribed', + external_ids: ['id2'], + emails: ['email2@example.com'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); + + it('should not include undefined fields in the output', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); +}); diff --git a/src/v0/destinations/braze/config.js b/src/v0/destinations/braze/config.js index 2bbade2754..8ccabbdb0a 100644 --- a/src/v0/destinations/braze/config.js +++ b/src/v0/destinations/braze/config.js @@ -36,7 +36,7 @@ const IDENTIFY_BRAZE_MAX_REQ_COUNT = 50; // https://www.braze.com/docs/api/endpoints/user_data/post_user_delete/ const ALIAS_BRAZE_MAX_REQ_COUNT = 50; -const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 50; +const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 25; const DEL_MAX_BATCH_SIZE = 50; const DESTINATION = 'braze'; diff --git a/src/v0/destinations/braze/transform.js b/src/v0/destinations/braze/transform.js index 09fb0205c5..2a9cb8e7dc 100644 --- a/src/v0/destinations/braze/transform.js +++ b/src/v0/destinations/braze/transform.js @@ -383,7 +383,7 @@ function processGroup(message, destination) { ); } subscriptionGroup.subscription_state = message.traits.subscriptionState; - subscriptionGroup.external_id = [message.userId]; + subscriptionGroup.external_ids = [message.userId]; const phone = getFieldValueFromMessage(message, 'phone'); const email = getFieldValueFromMessage(message, 'email'); if (phone) { diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js index 6c8cf64265..74cb7fb953 100644 --- a/src/v0/destinations/braze/util.js +++ b/src/v0/destinations/braze/util.js @@ -45,6 +45,42 @@ const getEndpointFromConfig = (destination) => { return endpoint; }; +// Merges external_ids, emails, and phones for entries with the same subscription_group_id and subscription_state +const combineSubscriptionGroups = (subscriptionGroups) => { + const uniqueGroups = {}; + + subscriptionGroups.forEach((group) => { + const key = `${group.subscription_group_id}-${group.subscription_state}`; + if (!uniqueGroups[key]) { + uniqueGroups[key] = { + ...group, + external_ids: [...(group.external_ids || [])], + emails: [...(group.emails || [])], + phones: [...(group.phones || [])], + }; + } else { + uniqueGroups[key].external_ids.push(...(group.external_ids || [])); + uniqueGroups[key].emails.push(...(group.emails || [])); + uniqueGroups[key].phones.push(...(group.phones || [])); + } + }); + + return Object.values(uniqueGroups).map((group) => { + const result = { + subscription_group_id: group.subscription_group_id, + subscription_state: group.subscription_state, + external_ids: [...new Set(group.external_ids)], + }; + if (group.emails?.length) { + result.emails = [...new Set(group.emails)]; + } + if (group.phones?.length) { + result.phones = [...new Set(group.phones)]; + } + return result; + }); +}; + const CustomAttributeOperationUtil = { customAttributeUpdateOperation(key, data, traits, mergeObjectsUpdateOperation) { data[key] = {}; @@ -381,8 +417,22 @@ function prepareGroupAndAliasBatch(arrayChunks, responseArray, destination, type } else if (type === 'subscription') { response.endpoint = getSubscriptionGroupEndPoint(getEndpointFromConfig(destination)); const subscription_groups = chunk; + // maketool transformed event + logger.info(`braze subscription chunk ${JSON.stringify(subscription_groups)}`); + + stats.gauge('braze_batch_subscription_size', subscription_groups.length, { + destination_id: destination.ID, + }); + + // Deduplicate the subscription groups before constructing the response body + const deduplicatedSubscriptionGroups = combineSubscriptionGroups(subscription_groups); + + stats.gauge('braze_batch_subscription_combined_size', deduplicatedSubscriptionGroups.length, { + destination_id: destination.ID, + }); + response.body.JSON = removeUndefinedAndNullValues({ - subscription_groups, + subscription_groups: deduplicatedSubscriptionGroups, }); } responseArray.push({ @@ -756,4 +806,5 @@ module.exports = { collectStatsForAliasFailure, collectStatsForAliasMissConfigurations, handleReservedProperties, + combineSubscriptionGroups, }; diff --git a/src/v0/destinations/google_adwords_offline_conversions/transform.js b/src/v0/destinations/google_adwords_offline_conversions/transform.js index 76b12587cd..2648f03e8a 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/transform.js +++ b/src/v0/destinations/google_adwords_offline_conversions/transform.js @@ -18,6 +18,7 @@ const { getClickConversionPayloadAndEndpoint, getConsentsDataFromIntegrationObj, getCallConversionPayload, + updateConversion, } = require('./utils'); const helper = require('./helper'); @@ -48,6 +49,9 @@ const getConversions = (message, metadata, { Config }, event, conversionType) => filteredCustomerId, eventLevelConsentsData, ); + convertedPayload.payload.conversions[0] = updateConversion( + convertedPayload.payload.conversions[0], + ); payload = convertedPayload.payload; endpoint = convertedPayload.endpoint; } else if (conversionType === 'store') { diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js index 89fe609df9..2d47095eea 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.js @@ -317,6 +317,35 @@ const getStoreConversionPayload = (message, Config, event) => { return payload; }; +const hasClickId = (conversion) => { + const { gbraid, wbraid, gclid } = conversion; + return gclid || wbraid || gbraid; +}; +const populateUserIdentifier = ({ email, phone, properties, payload, UserIdentifierSource }) => { + const copiedPayload = cloneDeep(payload); + // userIdentifierSource + // if userIdentifierSource doesn't exist in properties + // then it is taken from the webapp config + if (!properties.userIdentifierSource && UserIdentifierSource !== 'none') { + set( + copiedPayload, + 'conversions[0].userIdentifiers[0].userIdentifierSource', + UserIdentifierSource, + ); + // one of email or phone must be provided when none of gclid, wbraid and gbraid provided + } + if (!email && !phone) { + if (!hasClickId(copiedPayload.conversions[0])) { + throw new InstrumentationError( + `Either an email address or a phone number is required for user identification when none of gclid, wbraid, or gbraid is provided.`, + ); + } else { + // we are deleting userIdentifiers if any one of gclid, wbraid and gbraid is there but email or phone is not present + delete copiedPayload.conversions[0].userIdentifiers; + } + } + return copiedPayload; +}; const getClickConversionPayloadAndEndpoint = ( message, Config, @@ -335,7 +364,7 @@ const getClickConversionPayloadAndEndpoint = ( updatedClickMapping = removeHashToSha256TypeFromMappingJson(updatedClickMapping); } - const payload = constructPayload(message, updatedClickMapping); + let payload = constructPayload(message, updatedClickMapping); const endpoint = CLICK_CONVERSION.replace(':customerId', filteredCustomerId); @@ -353,17 +382,8 @@ const getClickConversionPayloadAndEndpoint = ( set(payload, 'conversions[0].cartData.items', itemList); } - // userIdentifierSource - // if userIdentifierSource doesn't exist in properties - // then it is taken from the webapp config - if (!properties.userIdentifierSource && UserIdentifierSource !== 'none') { - set(payload, 'conversions[0].userIdentifiers[0].userIdentifierSource', UserIdentifierSource); + payload = populateUserIdentifier({ email, phone, properties, payload, UserIdentifierSource }); - // one of email or phone must be provided - if (!email && !phone) { - throw new InstrumentationError(`Either of email or phone is required for user identifier`); - } - } // either of email or phone should be passed // defaultUserIdentifier depends on the webapp configuration // Ref - https://developers.google.com/google-ads/api/rest/reference/rest/v11/customers/uploadClickConversions#ClickConversion @@ -411,6 +431,25 @@ const getConsentsDataFromIntegrationObj = (message) => { return integrationObj?.consents || {}; }; +/** + * remove redundant ids + * @param {*} conversionCopy + */ +const updateConversion = (conversion) => { + const conversionCopy = cloneDeep(conversion); + if (conversionCopy.gclid) { + delete conversionCopy.wbraid; + delete conversionCopy.gbraid; + } + if (conversionCopy.wbraid && conversionCopy.gbraid) { + throw new InstrumentationError(`You can't use both wbraid and gbraid.`); + } + if (conversionCopy.wbraid || conversionCopy.gbraid) { + delete conversionCopy.userIdentifiers; + } + return conversionCopy; +}; + module.exports = { validateDestinationConfig, generateItemListFromProducts, @@ -423,4 +462,5 @@ module.exports = { getExisitingUserIdentifier, getConsentsDataFromIntegrationObj, getCallConversionPayload, + updateConversion, }; diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js index 2d1863413c..b6c6653782 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js @@ -244,7 +244,9 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { }; expect(() => getClickConversionPayloadAndEndpoint(fittingPayload, config, '9625812972'), - ).toThrow('Either of email or phone is required for user identifier'); + ).toThrow( + 'Either an email address or a phone number is required for user identification when none of gclid, wbraid, or gbraid is provided.', + ); }); it('finaliseConsent', () => { diff --git a/src/v0/destinations/mailjet/transform.js b/src/v0/destinations/mailjet/transform.js index 78b4f766d1..0a742b4cf7 100644 --- a/src/v0/destinations/mailjet/transform.js +++ b/src/v0/destinations/mailjet/transform.js @@ -1,5 +1,6 @@ const lodash = require('lodash'); const { TransformationError, InstrumentationError } = require('@rudderstack/integrations-lib'); +const stats = require('../../../util/stats'); const { getSuccessRespEvents, defaultRequestConfig, @@ -105,6 +106,9 @@ const batchEvents = (successRespList) => { const eventChunks = lodash.chunk(eventGroups[combination], MAX_BATCH_SIZE); // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] eventChunks.forEach((chunk) => { + stats.gauge('mailjet_packing_size', chunk.length, { + group: combination, + }); const batchEventResponse = generateBatchedPaylaodForArray(chunk, combination); batchedResponseList.push( getSuccessRespEvents( diff --git a/src/v0/destinations/snowpipe_streaming/transform.js b/src/v0/destinations/snowpipe_streaming/transform.js index 3b0a6e1a93..36fe19ceff 100644 --- a/src/v0/destinations/snowpipe_streaming/transform.js +++ b/src/v0/destinations/snowpipe_streaming/transform.js @@ -1 +1,26 @@ -module.exports = require('../snowflake/transform'); +const transform = require('../snowflake/transform'); +const { processWarehouseMessage } = require('../../../warehouse'); + +const provider = 'snowpipe_streaming'; + +function process(event) { + const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; + const whStoreEvent = event.destination.Config.storeFullEvent === true; + const destJsonPaths = event.destination?.Config?.jsonPaths || ''; + return processWarehouseMessage(event.message, { + metadata: event.metadata, + whSchemaVersion, + whStoreEvent, + getDataTypeOverride: transform.getDataTypeOverride, + provider, + sourceCategory: event.metadata ? event.metadata.sourceCategory : null, + destJsonPaths, + destConfig: event.destination?.Config, + }); +} + +module.exports = { + provider, + process, + getDataTypeOverride: transform.getDataTypeOverride, +}; diff --git a/src/v0/util/facebookUtils/networkHandler.js b/src/v0/util/facebookUtils/networkHandler.js index fbb7899efe..46ac59a07a 100644 --- a/src/v0/util/facebookUtils/networkHandler.js +++ b/src/v0/util/facebookUtils/networkHandler.js @@ -85,7 +85,10 @@ const errorDetailsMap = { "Object with ID 'PIXEL_ID' / 'DATASET_ID' / 'AUDIENCE_ID' does not exist, cannot be loaded due to missing permissions, or does not support this operation", ) .build(), - default: new ErrorDetailsExtractorBuilder().setStatus(400).setMessageField('message').build(), + default: new ErrorDetailsExtractorBuilder() + .setStatus(400) + .setMessageField('error_user_msg') + .build(), }, 1: { // An unknown error occurred. @@ -107,9 +110,7 @@ const errorDetailsMap = { .setStat({ [TAG_NAMES.ERROR_TYPE]: ERROR_TYPES.AUTH, }) - .setMessage( - 'The session has been invalidated because the user changed their password or Facebook has changed the session for security reasons', - ) + .setMessageField('error_user_msg') .build(), 463: new ErrorDetailsExtractorBuilder() @@ -117,56 +118,56 @@ const errorDetailsMap = { .setStat({ [TAG_NAMES.ERROR_TYPE]: ERROR_TYPES.AUTH, }) - .setMessageField('message') + .setMessageField('error_user_msg') .build(), default: new ErrorDetailsExtractorBuilder() .setStatus(400) .setStat({ [TAG_NAMES.ERROR_TYPE]: ERROR_TYPES.AUTH, }) - .setMessage('Invalid OAuth 2.0 access token') + .setMessageField('error_user_msg') .build(), }, 3: { default: new ErrorDetailsExtractorBuilder() .setStatus(400) - .setMessage('Capability or permissions issue.') + .setMessageField('error_user_msg') .build(), }, 2: { default: new ErrorDetailsExtractorBuilder() .setStatus(500) - .setMessage('Temporary issue due to downtime.') + .setMessageField('error_user_msg') .build(), }, 341: { default: new ErrorDetailsExtractorBuilder() .setStatus(500) - .setMessage('Application limit reached: Temporary issue due to downtime or throttling') + .setMessageField('error_user_msg') .build(), }, 368: { default: new ErrorDetailsExtractorBuilder() .setStatus(500) - .setMessage('Temporarily blocked for policies violations.') + .setMessageField('error_user_msg') .build(), }, 5000: { default: new ErrorDetailsExtractorBuilder() .setStatus(500) - .setMessage('Unknown Error Code') + .setMessageField('error_user_msg') .build(), }, 4: { default: new ErrorDetailsExtractorBuilder() .setStatus(429) - .setMessage('API Too Many Calls') + .setMessageField('error_user_msg') .build(), }, 17: { default: new ErrorDetailsExtractorBuilder() .setStatus(429) - .setMessage('API User Too Many Calls') + .setMessageField('error_user_msg') .build(), }, // facebook custom audience related error codes @@ -176,9 +177,7 @@ const errorDetailsMap = { 294: { default: new ErrorDetailsExtractorBuilder() .setStatus(400) - .setMessage( - 'Missing permission. Please make sure you have ads_management permission and the application is included in the allowlist', - ) + .setMessageField('error_user_msg') .build(), }, 1487301: { @@ -214,7 +213,10 @@ const errorDetailsMap = { .build(), }, 200: { - default: new ErrorDetailsExtractorBuilder().setStatus(403).setMessageField('message').build(), + default: new ErrorDetailsExtractorBuilder() + .setStatus(403) + .setMessageField('error_user_msg') + .build(), }, 21009: { default: new ErrorDetailsExtractorBuilder().setStatus(500).setMessageField('message').build(), @@ -236,9 +238,11 @@ const getStatus = (error) => { const isErrorDetailEmpty = isEmpty(errorDetail); if (isErrorDetailEmpty) { // Unhandled error response + const errorMessage = get(error?.error || error, 'error_user_msg'); return { status: errorStatus, stats: { [TAG_NAMES.META]: METADATA.UNHANDLED_STATUS_CODE }, + errorMessage: errorMessage || JSON.stringify(error), }; } errorStatus = errorDetail.status; diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js index c31bc74bf1..292a980a93 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js @@ -1,18 +1,9 @@ /* eslint-disable @typescript-eslint/naming-convention */ const lodash = require('lodash'); const get = require('get-value'); -// const { RedisError } = require('@rudderstack/integrations-lib'); const stats = require('../../../../util/stats'); -const { - getShopifyTopic, - // createPropertiesForEcomEvent, - extractEmailFromPayload, - getAnonymousIdAndSessionId, - // getHashLineItems, -} = require('../../../../v0/sources/shopify/util'); -// const logger = require('../../../logger'); +const { getShopifyTopic, extractEmailFromPayload } = require('../../../../v0/sources/shopify/util'); const { removeUndefinedAndNullValues, isDefinedAndNotNull } = require('../../../../v0/util'); -// const { RedisDB } = require('../../../util/redis/redisConnector'); const Message = require('../../../../v0/sources/message'); const { EventType } = require('../../../../constants'); const { @@ -28,6 +19,7 @@ const { const { createPropertiesForEcomEventFromWebhook, getProductsFromLineItems, + getAnonymousIdFromAttributes, } = require('./serverSideUtlis'); const NO_OPERATION_SUCCESS = { @@ -128,12 +120,9 @@ const processEvent = async (inputEvent, metricMetadata) => { message.setProperty('traits.email', email); } } + // attach anonymousId if the event is track event using note_attributes if (message.type !== EventType.IDENTIFY) { - const { anonymousId } = await getAnonymousIdAndSessionId( - message, - { shopifyTopic, ...metricMetadata }, - null, - ); + const anonymousId = getAnonymousIdFromAttributes(event); if (isDefinedAndNotNull(anonymousId)) { message.setProperty('anonymousId', anonymousId); } diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js index a611d1d8dc..4a2839103b 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js @@ -1,6 +1,7 @@ const { getProductsFromLineItems, createPropertiesForEcomEventFromWebhook, + getAnonymousIdFromAttributes, } = require('./serverSideUtlis'); const { constructPayload } = require('../../../../v0/util'); @@ -109,4 +110,21 @@ describe('serverSideUtils.js', () => { }); }); }); + + describe('getAnonymousIdFromAttributes', () => { + // Handles empty note_attributes array gracefully + it('should return null when note_attributes is an empty array', async () => { + const event = { note_attributes: [] }; + const result = await getAnonymousIdFromAttributes(event); + expect(result).toBeNull(); + }); + + it('get anonymousId from noteAttributes', async () => { + const event = { + note_attributes: [{ name: 'rudderAnonymousId', value: '123456' }], + }; + const result = await getAnonymousIdFromAttributes(event); + expect(result).toEqual('123456'); + }); + }); }); diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js index eed03de71f..951fa479e4 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -1,5 +1,5 @@ +const { isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); const { constructPayload } = require('../../../../v0/util'); - const { lineItemsMappingJSON, productMappingJSON, @@ -17,7 +17,6 @@ const getProductsFromLineItems = (lineItems, mapping) => { } const products = []; lineItems.forEach((lineItem) => { - // const product = constructPayload(lineItem, lineItemsMappingJSON); const product = constructPayload(lineItem, mapping); products.push(product); }); @@ -39,7 +38,24 @@ const createPropertiesForEcomEventFromWebhook = (message) => { return mappedPayload; }; +/** + * Returns the anonymousId from the noteAttributes array in the webhook event + * @param {Object} event + * @returns {String} anonymousId + */ +const getAnonymousIdFromAttributes = (event) => { + if (!isDefinedAndNotNull(event) || !isDefinedAndNotNull(event.note_attributes)) { + return null; // Return early if event or note_attributes is invalid + } + + const noteAttributes = event.note_attributes; + const rudderAnonymousIdObject = noteAttributes.find((attr) => attr.name === 'rudderAnonymousId'); + + return rudderAnonymousIdObject ? rudderAnonymousIdObject.value : null; +}; + module.exports = { createPropertiesForEcomEventFromWebhook, getProductsFromLineItems, + getAnonymousIdFromAttributes, }; diff --git a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js index 0c1007f311..46ae59e0cf 100644 --- a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js @@ -14,7 +14,7 @@ const { function getNestedValue(object, path) { const keys = path.split('.'); - return keys.reduce((nestedObject, key) => nestedObject && nestedObject[key], object); + return keys.reduce((nestedObject, key) => nestedObject?.[key], object); } function setNestedValue(object, path, value) { diff --git a/src/warehouse/config/ReservedKeywords.json b/src/warehouse/config/ReservedKeywords.json index 3c383d24bf..53b8738e1d 100644 --- a/src/warehouse/config/ReservedKeywords.json +++ b/src/warehouse/config/ReservedKeywords.json @@ -1519,6 +1519,99 @@ "WHERE": true, "WITH": true }, + "SNOWPIPE_STREAMING": { + "ACCOUNT": true, + "ALL": true, + "ALTER": true, + "AND": true, + "ANY": true, + "AS": true, + "BETWEEN": true, + "BY": true, + "CASE": true, + "CAST": true, + "CHECK": true, + "COLUMN": true, + "CONNECT": true, + "CONNECTION": true, + "CONSTRAINT": true, + "CREATE": true, + "CROSS": true, + "CURRENT": true, + "CURRENT_DATE": true, + "CURRENT_TIME": true, + "CURRENT_TIMESTAMP": true, + "CURRENT_USER": true, + "DATABASE": true, + "DELETE": true, + "DISTINCT": true, + "DROP": true, + "ELSE": true, + "EXISTS": true, + "FALSE": true, + "FOLLOWING": true, + "FOR": true, + "FROM": true, + "FULL": true, + "GRANT": true, + "GROUP": true, + "GSCLUSTER": true, + "HAVING": true, + "ILIKE": true, + "IN": true, + "INCREMENT": true, + "INNER": true, + "INSERT": true, + "INTERSECT": true, + "INTO": true, + "IS": true, + "ISSUE": true, + "JOIN": true, + "LATERAL": true, + "LEFT": true, + "LIKE": true, + "LOCALTIME": true, + "LOCALTIMESTAMP": true, + "MINUS": true, + "NATURAL": true, + "NOT": true, + "NULL": true, + "OF": true, + "ON": true, + "OR": true, + "ORDER": true, + "ORGANIZATION": true, + "QUALIFY": true, + "REGEXP": true, + "REVOKE": true, + "RIGHT": true, + "RLIKE": true, + "ROW": true, + "ROWS": true, + "SAMPLE": true, + "SCHEMA": true, + "SELECT": true, + "SET": true, + "SOME": true, + "START": true, + "TABLE": true, + "TABLESAMPLE": true, + "THEN": true, + "TO": true, + "TRIGGER": true, + "TRUE": true, + "TRY_CAST": true, + "UNION": true, + "UNIQUE": true, + "UPDATE": true, + "USING": true, + "VALUES": true, + "VIEW": true, + "WHEN": true, + "WHENEVER": true, + "WHERE": true, + "WITH": true + }, "CLICKHOUSE": {}, "S3_DATALAKE": { "ALL": true, diff --git a/src/warehouse/index.js b/src/warehouse/index.js index 3491a257da..ea663c9b2f 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -349,7 +349,10 @@ function stringLikeObjectToString(obj) { */ function getColumns(options, event, columnTypes) { const columns = {}; - const uuidTS = options.provider === 'snowflake' ? 'UUID_TS' : 'uuid_ts'; + const uuidTS = + options.provider === 'snowflake' || options.provider === 'snowpipe_streaming' + ? 'UUID_TS' + : 'uuid_ts'; columns[uuidTS] = 'datetime'; // add loaded_at for bq to be segment compatible if (options.provider === 'bq') { @@ -377,6 +380,7 @@ function getColumns(options, event, columnTypes) { const fullEventColumnTypeByProvider = { snowflake: 'json', + snowpipe_streaming: 'json', rs: 'text', bq: 'string', postgres: 'json', @@ -613,6 +617,15 @@ function enhanceContextWithSourceDestInfo(message, metadata) { message.context = context; } +function shouldSkipUsersTable(options) { + return ( + options.provider === 'snowpipe_streaming' || + options.destConfig?.skipUsersTable || + options.integrationOptions?.skipUsersTable || + false + ); +} + function processWarehouseMessage(message, options) { const utils = getVersionedUtils(options.whSchemaVersion); options.utils = utils; @@ -638,8 +651,7 @@ function processWarehouseMessage(message, options) { const eventType = message.type?.toLowerCase(); const skipTracksTable = options.destConfig?.skipTracksTable || options.integrationOptions.skipTracksTable || false; - const skipUsersTable = - options.destConfig?.skipUsersTable || options.integrationOptions.skipUsersTable || false; + const skipUsersTable = shouldSkipUsersTable(options); const skipReservedKeywordsEscaping = options.integrationOptions.skipReservedKeywordsEscaping || false; diff --git a/src/warehouse/v1/util.js b/src/warehouse/v1/util.js index d1289bc674..95dd59b030 100644 --- a/src/warehouse/v1/util.js +++ b/src/warehouse/v1/util.js @@ -11,7 +11,7 @@ function safeTableName(options, name = '') { if (tableName === '') { throw new TransformationError('Table name cannot be empty.'); } - if (provider === 'snowflake') { + if (provider === 'snowflake' || provider === 'snowpipe_streaming') { tableName = tableName.toUpperCase(); } else if (provider === 'postgres') { tableName = tableName.substr(0, 63); @@ -41,7 +41,7 @@ function safeColumnName(options, name = '') { if (columnName === '') { throw new TransformationError('Column name cannot be empty.'); } - if (provider === 'snowflake') { + if (provider === 'snowflake' || provider === 'snowpipe_streaming') { columnName = columnName.toUpperCase(); } else if (provider === 'postgres') { columnName = columnName.substr(0, 63); diff --git a/test/__tests__/data/warehouse/events.js b/test/__tests__/data/warehouse/events.js index bca6f776be..99343e43e1 100644 --- a/test/__tests__/data/warehouse/events.js +++ b/test/__tests__/data/warehouse/events.js @@ -368,6 +368,146 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "TRACKS", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "button clicked", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + EVENT: "button_clicked" + } + }, + { + metadata: { + table: "BUTTON_CLICKED", + columns: { + UUID_TS: "datetime", + CURRENCY: "string", + REVENUE: "int", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CURRENCY: "USD", + REVENUE: 50, + STACK_HISTORY_ERROR_DETAILS: [ + { + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" + } + ], + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "button clicked", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + EVENT: "button_clicked", + } + } + ], s3_datalake: [ { metadata: { @@ -845,6 +985,78 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "IDENTIFIES", + columns: { + UUID_TS: "datetime", + CITY: "string", + COUNTRY: "string", + EMAIL: "string", + FIRSTNAME: "string", + LASTNAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CITY: "Disney", + COUNTRY: "UK", + EMAIL: "mickey@disney.com", + FIRSTNAME: "Mickey", + LASTNAME: "Mouse", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "2536eda4-d638-4c93-8014-8ffe3f083214", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.363Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.362Z", + CHANNEL: "web" + } + }, + ], s3_datalake: [ { metadata: { @@ -1170,6 +1382,66 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "PAGES", + columns: { + UUID_TS: "datetime", + PATH: "string", + URL: "string", + NAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_USER_AGENT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + PATH: "/tests/html/index2.html", + URL: "http://localhost/tests/html/index2.html", + NAME: "sample title", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "dd266c67-9199-4a52-ba32-f46ddde67312", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.359Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.358Z", + CHANNEL: "web" + } + } + ], s3_datalake: [ { metadata: { @@ -1424,6 +1696,67 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "SCREENS", + columns: { + UUID_TS: "datetime", + PATH: "string", + CATEGORY: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + TIMESTAMP: "datetime", + CHANNEL: "string", + NAME: "string" + } + }, + data: { + PATH: "/abc", + CATEGORY: "test-category", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.0", + CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.0", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + CONTEXT_LOCALE: "en-US", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + ID: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + ANONYMOUS_ID: "00000000000000000000000000", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2019-10-14T11:15:53.296Z", + ORIGINAL_TIMESTAMP: "2019-10-14T11:15:18.299Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + CHANNEL: "web", + NAME: "ApplicationLoaded" + } + } + ], s3_datalake: [ { metadata: { @@ -1687,6 +2020,73 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "GROUPS", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_LOCALE: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_OS_ID: "string", + CONTEXT_OS_MANUFACTURER: "string", + CONTEXT_OS_MODEL: "string", + CONTEXT_OS_TYPE: "string", + CONTEXT_SCREEN_DENSITY: "int", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + GROUP_ID: "string", + SENT_AT: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + TIMESTAMP: "datetime", + CHANNEL: "string" + } + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.0", + CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.0", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + CONTEXT_LOCALE: "en-US", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_OS_ID: "72e528f869711c3d", + CONTEXT_OS_MANUFACTURER: "Google", + CONTEXT_OS_MODEL: "sdk_gphone_x86", + CONTEXT_OS_NAME: "", + CONTEXT_OS_TOKEN: "", + CONTEXT_OS_TYPE: "android", + CONTEXT_SCREEN_DENSITY: 2, + ID: "84e26acc-56a5-4835-8233-591137fca468", + ANONYMOUS_ID: "00000000000000000000000000", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + GROUP_ID: "g1", + SENT_AT: "2019-10-14T09:03:22.563Z", + ORIGINAL_TIMESTAMP: "2019-10-14T09:03:17.562Z", + TIMESTAMP: "2020-01-24T11:59:02.403Z", + RECEIVED_AT: "2020-01-24T11:59:02.403Z", + CHANNEL: "web" + } + } + ], s3_datalake: [ { metadata: { @@ -1956,6 +2356,70 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "ALIASES", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + PREVIOUS_ID: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "79313729-7fe5-4204-963a-dc46f4205e4e", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "1234abc", + SENT_AT: "2020-01-24T06:29:02.366Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.366Z", + CHANNEL: "web", + PREVIOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" + } + } + ], s3_datalake: [ { metadata: { @@ -2147,6 +2611,40 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "EXTRACT_EVENT", + columns: { + DATE_PROPERTY: "datetime", + DATE_PROPERTY_2: "datetime", + CONTEXT_SOURCES_JOB_ID: "string", + CONTEXT_SOURCES_JOB_RUN_ID: "string", + CONTEXT_SOURCES_TASK_RUN_ID: "string", + CONTEXT_SOURCES_VERSION: "string", + EVENT: "string", + ID: "string", + USER_ID: "string", + RECEIVED_AT: "datetime", + BOOLEAN_PROPERTY: "boolean", + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + DATE_PROPERTY: "2023-02-01T07:53:31.430Z", + DATE_PROPERTY_2: "2023-01-31T07:39:10.002Z", + CONTEXT_SOURCES_JOB_ID: "2JABSy1nq89H7xeJimBL2pCtOOp", + CONTEXT_SOURCES_JOB_RUN_ID: "cfd6705nsevh5p2l77ag", + CONTEXT_SOURCES_TASK_RUN_ID: "cfd6705nsevh5p2l77b0", + CONTEXT_SOURCES_VERSION: "version", + ID: "some-uuid", + USER_ID: "dummy-user-id-inside-properties", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + EVENT: "extract_event", + BOOLEAN_PROPERTY: true, + } + } + ], s3_datalake: [ { metadata: { @@ -2193,6 +2691,9 @@ function output(eventType, provider) { if (provider === "snowflake") { return _.cloneDeep(sampleEvents[eventType].output.snowflake); } + if (provider === "snowpipe_streaming") { + return _.cloneDeep(sampleEvents[eventType].output.snowpipe_streaming); + } if (provider === "s3_datalake") { return _.cloneDeep(sampleEvents[eventType].output.s3_datalake); } diff --git a/test/__tests__/data/warehouse/integration_options_events.js b/test/__tests__/data/warehouse/integration_options_events.js index 35eafb6a9b..b3f076851b 100644 --- a/test/__tests__/data/warehouse/integration_options_events.js +++ b/test/__tests__/data/warehouse/integration_options_events.js @@ -74,6 +74,13 @@ const sampleEvents = { jsonPaths: ["tMap"] } }, + SNOWPIPE_STREAMING: { + options: { + skipTracksTable: true, + useBlendoCasing: true, + jsonPaths: ["tMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -432,6 +439,88 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "_GROUPS", + columns: { + UUID_TS: "datetime", + CURRENCY: "string", + REVENUE: "int", + PATH_TO_$1_000_000: "string", + _9OMEGA: "boolean", + CAMELCASE123KEY: "string", + TESTMAP_NESTEDMAP: "json", + TMAP: "json", + TESTARRAY: "json", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USERAGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CURRENCY: "USD", + REVENUE: 50, + PATH_TO_$1_000_000: "None", + _9OMEGA: true, + CAMELCASE123KEY: "camel case", + TESTMAP_NESTEDMAP: '{"n1":"nested prop 1"}', + TMAP: '{"t1":10,"t2":20}', + TESTARRAY: '["This is","an","array"]', + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USERAGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "groups", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + EVENT: "groups" + } + } + ], s3_datalake: [ { metadata: { @@ -730,7 +819,7 @@ const sampleEvents = { ] } }, - users: { + identify: { input: { destination: { Config: { @@ -1255,6 +1344,75 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + ], s3_datalake: [ { "data": { @@ -1639,6 +1797,8 @@ function opOutput(eventType, provider) { switch (provider) { case "snowflake": return _.cloneDeep(sampleEvents[eventType].output.snowflake); + case "snowpipe_streaming": + return _.cloneDeep(sampleEvents[eventType].output.snowpipe_streaming); case "s3_datalake": return _.cloneDeep(sampleEvents[eventType].output.s3_datalake); case "rs": @@ -1646,13 +1806,13 @@ function opOutput(eventType, provider) { case "bq": return _.cloneDeep(sampleEvents[eventType].output.bq); case "gcs_datalake": - if (eventType === 'users') { + if (eventType === 'identify') { return _.cloneDeep(sampleEvents[eventType].output.gcs_datalake); } else { return _.cloneDeep(sampleEvents[eventType].output.default); } case "azure_datalake": - if (eventType === 'users') { + if (eventType === 'identify') { return _.cloneDeep(sampleEvents[eventType].output.azure_datalake); } else { return _.cloneDeep(sampleEvents[eventType].output.default); diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js index 30cce51fb7..7615abe5e1 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js @@ -78,6 +78,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -426,5 +434,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "ALIASES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js index 9d38ecc292..b28b3a161e 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js @@ -59,6 +59,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -276,6 +284,43 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_SOURCES_JOB_ID": "djfhksdjhfkjdhfkjahkf", + "CONTEXT_SOURCES_JOB_RUN_ID": "job_run_id", + "CONTEXT_SOURCES_TASK_RUN_ID": "task_run_id", + "CONTEXT_SOURCES_VERSION": "1169/merge", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EVENT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "P_MAP_NESTED_MAP_N_1": "nested prop 1", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50 + }, + "metadata": { + "columns": { + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_SOURCES_JOB_ID": "string", + "CONTEXT_SOURCES_JOB_RUN_ID": "string", + "CONTEXT_SOURCES_TASK_RUN_ID": "string", + "CONTEXT_SOURCES_VERSION": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EVENT": "string", + "ID": "string", + "P_MAP_NESTED_MAP_N_1": "string", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js index bbae993b27..457d251a74 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js @@ -78,6 +78,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, GCS_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -431,5 +439,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "GROUPS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js index 42f5cccf49..d91ef5f44b 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js @@ -53,6 +53,16 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "CTMap.nestedMap", + "TMap.nestedMap", + "CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -879,6 +889,85 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP_N_1": "nested prop 1", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "CT_MAP_NESTED_MAP_N_1": "nested prop 1", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "T_MAP_NESTED_MAP_N_1": "nested prop 1", + "UP_MAP_NESTED_MAP_N_1": "nested prop 1", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "CT_MAP_NESTED_MAP_N_1": "string", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "T_MAP_NESTED_MAP_N_1": "string", + "UP_MAP_NESTED_MAP_N_1": "string", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + ], datalake: [ { "data": { diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js index dc2d3a3318..322ee8f3e6 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js @@ -67,6 +67,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -404,5 +412,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PAGES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js index 9b789d2747..27d7b9a69b 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js @@ -67,6 +67,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + ".ctestMap.cnestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -404,5 +412,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "SCREENS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js index 691ce57ca1..aabfce149b 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js @@ -78,6 +78,15 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -828,6 +837,148 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "TRACKS" + } + }, + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EMAIL": "test@gmail.com", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EMAIL": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js index d5d57f85b9..4b74fd3768 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js @@ -66,6 +66,11 @@ module.exports = { jsonPaths: ["alias.traits.testMap.nestedMap", "alias.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["alias.traits.testMap.nestedMap", "alias.context.ctestMap.cnestedMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -414,5 +419,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "ALIASES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js index a950b7f526..03745c0502 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js @@ -59,6 +59,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "extract.properties.PMap.nestedMap", + "extract.context.CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -276,6 +284,43 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_SOURCES_JOB_ID": "djfhksdjhfkjdhfkjahkf", + "CONTEXT_SOURCES_JOB_RUN_ID": "job_run_id", + "CONTEXT_SOURCES_TASK_RUN_ID": "task_run_id", + "CONTEXT_SOURCES_VERSION": "1169/merge", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EVENT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50 + }, + "metadata": { + "columns": { + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_SOURCES_JOB_ID": "string", + "CONTEXT_SOURCES_JOB_RUN_ID": "string", + "CONTEXT_SOURCES_TASK_RUN_ID": "string", + "CONTEXT_SOURCES_VERSION": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EVENT": "string", + "ID": "string", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js index 6979aa1100..1e7c8110da 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js @@ -66,6 +66,11 @@ module.exports = { jsonPaths: ["group.traits.testMap.nestedMap", "group.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["group.traits.testMap.nestedMap", "group.context.ctestMap.cnestedMap"] + } + }, GCS_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -419,5 +424,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "GROUPS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js index 89fcc23cd5..d9c3b80df3 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js @@ -53,6 +53,16 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "identify.userProperties.UPMap.nestedMap", + "identify.context.traits.CTMap.nestedMap", + "identify.traits.TMap.nestedMap", + "identify.context.CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -879,6 +889,85 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "CT_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "T_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP": "json", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "CT_MAP_NESTED_MAP": "json", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "T_MAP_NESTED_MAP": "json", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + ], datalake: [ { "data": { diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js index d5282c1cfd..1f72ec31cf 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js @@ -55,6 +55,11 @@ module.exports = { jsonPaths: ["page.properties.testMap.nestedMap", "page.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["page.properties.testMap.nestedMap", "page.context.ctestMap.cnestedMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -392,5 +397,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PAGES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js index 3eec47b89d..68b5aafcb5 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js @@ -55,6 +55,11 @@ module.exports = { jsonPaths: ["screen.properties.testMap.nestedMap", "screen.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["screen.properties.testMap.nestedMap", "screen.context.ctestMap.cnestedMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -392,5 +397,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "SCREENS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js index 6b411835db..70c20b8b8e 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js @@ -78,6 +78,15 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "track.userProperties.UPMap.nestedMap", + "track.properties.PMap.nestedMap", + "track.context.CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -828,6 +837,148 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "TRACKS" + } + }, + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EMAIL": "test@gmail.com", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EMAIL": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/names.js b/test/__tests__/data/warehouse/names.js index b0380dc06d..e6015cef2c 100644 --- a/test/__tests__/data/warehouse/names.js +++ b/test/__tests__/data/warehouse/names.js @@ -41,6 +41,19 @@ const names = { C_Z: "string", CAMEL_CASE_123_KEY: "string", _1_C_COMEGA: "string" + }, + snowpipe_streaming: { + OMEGA: "string", + OMEGA_V_2: "string", + _9_MEGA: "string", + MEGA: "string", + OME_GA: "string", + ALPHA: "string", + OME_GA: "string", + _9_MEGA_90: "string", + C_Z: "string", + CAMEL_CASE_123_KEY: "string", + _1_C_COMEGA: "string" } }, data: { @@ -69,6 +82,19 @@ const names = { C_Z: "test", CAMEL_CASE_123_KEY: "test", _1_C_COMEGA: "test" + }, + snowpipe_streaming: { + OMEGA: "test", + OMEGA_V_2: "test", + _9_MEGA: "test", + MEGA: "test", + OME_GA: "test", + ALPHA: "test", + OME_GA: "test", + _9_MEGA_90: "test", + C_Z: "test", + CAMEL_CASE_123_KEY: "test", + _1_C_COMEGA: "test" } }, namesMap: { @@ -97,6 +123,19 @@ const names = { Cízǔ: "C_Z", CamelCase123Key: "CAMEL_CASE_123_KEY", "1CComega": "_1_C_COMEGA" + }, + snowpipe_streaming: { + omega: "OMEGA", + "omega v2": "OMEGA_V_2", + "9mega": "_9_MEGA", + "mega&": "MEGA", + ome$ga: "OME_GA", + alpha$: "ALPHA", + "ome_ ga": "OME_GA", + "9mega________-________90": "_9_MEGA_90", + Cízǔ: "C_Z", + CamelCase123Key: "CAMEL_CASE_123_KEY", + "1CComega": "_1_C_COMEGA" } } } diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 7fdecbd7cf..44d1d33ff8 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -37,14 +37,6 @@ const integrations = [ "s3_datalake", "gcs_datalake", ]; - -const integration = (index ) => { - const it = integrations[index]; - if (it === "snowflake" || it === "snowpipe_streaming") { - return "snowflake"; - } - return it; -} const transformers = integrations.map(integration => require(`../../src/${version}/destinations/${integration}/transform`) ); @@ -62,7 +54,7 @@ const propsKeyMap = { }; const integrationCasedString = (integration, str) => { - if (integration === "snowflake") { + if (integration === "snowflake" || integration === "snowpipe_streaming") { return str.toUpperCase(); } return str; @@ -74,7 +66,7 @@ describe("event types", () => { const i = input("track"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("track", integration(index))); + expect(received).toMatchObject(output("track", integrations[index])); }); }); }); @@ -85,7 +77,7 @@ describe("event types", () => { // also verfies priority order between traits and context.traits transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("identify", integration(index))); + expect(received).toMatchObject(output("identify", integrations[index])); }); }); }); @@ -95,7 +87,7 @@ describe("event types", () => { const i = input("page"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("page", integration(index))); + expect(received).toMatchObject(output("page", integrations[index])); }); }); it("should take name from properties if top-level name is missing", () => { @@ -104,7 +96,7 @@ describe("event types", () => { delete i.message.name; transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("page", integration(index))); + expect(received).toMatchObject(output("page", integrations[index])); }); }); }); @@ -114,7 +106,7 @@ describe("event types", () => { const i = input("screen"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integration(index))); + expect(received).toMatchObject(output("screen", integrations[index])); }); }); it("should take name from properties if top-level name is missing", () => { @@ -123,7 +115,7 @@ describe("event types", () => { delete i.message.name; transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integration(index))); + expect(received).toMatchObject(output("screen", integrations[index])); }); }); }); @@ -133,7 +125,7 @@ describe("event types", () => { const i = input("alias"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("alias", integration(index))); + expect(received).toMatchObject(output("alias", integrations[index])); }); }); }); @@ -143,7 +135,7 @@ describe("event types", () => { const i = input("extract"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("extract", integration(index))); + expect(received).toMatchObject(output("extract", integrations[index])); }); }); }); @@ -161,8 +153,9 @@ describe("column & table names", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); - const provider = - (integration(index) === "snowflake" || integration(index) == "snowpipe_streaming") ? "snowflake" : "default"; + const provider = ["snowflake", "snowpipe_streaming"].includes(integrations[index]) + ? integrations[index] + : "default"; expect(received[1].metadata.columns).toMatchObject( names.output.columns[provider] @@ -195,7 +188,7 @@ describe("column & table names", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); - if (integration(index) === "postgres") { + if (integrations[index] === "postgres") { expect(received[1].metadata).toHaveProperty( "table", "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1" @@ -219,7 +212,7 @@ describe("column & table names", () => { //KEY should be trimmed to 63 return; } - if (integration(index) === "snowflake") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { expect(received[1].metadata).toHaveProperty( "table", "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2" @@ -242,7 +235,7 @@ describe("column & table names", () => { ); return; } - if (integration(index) === "s3_datalake" || integration(index) === "gcs_datalake" || integration(index) === "azure_datalake") { + if (integrations[index] === "s3_datalake" || integrations[index] === "gcs_datalake" || integrations[index] === "azure_datalake") { expect(received[1].metadata).toHaveProperty( "table", "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5" @@ -325,7 +318,7 @@ describe("conflict between rudder set props and user set props", () => { const propsKey = propsKeyMap[evType]; transformers.forEach((transformer, index) => { let sampleRudderPropKey = "id"; - if (integration(index) === "snowflake") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { sampleRudderPropKey = "ID"; } @@ -363,7 +356,7 @@ describe("handle reserved words", () => { const propsKey = propsKeyMap[evType]; transformers.forEach((transformer, index) => { const reserverdKeywordsMap = - reservedANSIKeywordsMap[integration(index).toUpperCase()]; + reservedANSIKeywordsMap[integrations[index].toUpperCase()]; i.message[propsKey] = Object.assign( i.message[propsKey] || {}, @@ -373,7 +366,7 @@ describe("handle reserved words", () => { const received = transformer.process(i); const out = - evType === "track" || evType === "identify" + evType === "track" || (evType === "identify" && integrations[index] !== 'snowpipe_streaming') ? received[1] : received[0]; @@ -386,7 +379,7 @@ describe("handle reserved words", () => { } else { k = snakeCasedKey; } - if (integration(index) === "snowflake") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { expect(out.metadata.columns).toHaveProperty(k); } else { expect(out.metadata.columns).toHaveProperty(k.toLowerCase()); @@ -470,24 +463,24 @@ describe("context ip", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[0].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("new_ip"); if (received[1]) { expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[1].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("new_ip"); } @@ -505,23 +498,23 @@ describe("context ip", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[0].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("requested_ip"); if (received[1]) { expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[1].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("requested_ip"); } @@ -542,10 +535,10 @@ describe("remove rudder property if rudder property is null", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ); }); }); @@ -560,29 +553,29 @@ describe("remove any property if event is object ", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); }); transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "event_text") + integrationCasedString(integrations[index], "event_text") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "event_text") + integrationCasedString(integrations[index], "event_text") ); }); i.message.channel = { channel: "android" }; transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); }); }); @@ -598,13 +591,13 @@ describe("store full rudder event", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); const columnName = integrationCasedString( - integration(index), + integrations[index], "rudder_event" ); expect(received[0].metadata.columns).toHaveProperty(columnName); expect(received[0].metadata.columns[columnName]).toEqual( - fullEventColumnTypeByProvider[integration(index)] + fullEventColumnTypeByProvider[integrations[index]] ); expect(received[0].data[columnName]).toEqual(JSON.stringify(i.message)); @@ -644,7 +637,7 @@ describe("rudder reserved columns", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); checkProps.forEach(k => { - k = integrationCasedString(integration(index), k); + k = integrationCasedString(integrations[index], k); expect(received[0].metadata.columns).not.toHaveProperty(k); expect(received[0].data).not.toHaveProperty(k); if (received[1]) { @@ -664,14 +657,25 @@ describe("id column datatype for users table", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); + if (integrations[index] === 'snowpipe_streaming') { + expect(received).toHaveLength(1); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "user_id") + ] + ).toEqual("int"); + return; + } + + expect(received).toHaveLength(2); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "user_id") + integrationCasedString(integrations[index], "user_id") ] ).toEqual("int"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); }); @@ -681,14 +685,25 @@ describe("id column datatype for users table", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); + if (integrations[index] === 'snowpipe_streaming') { + expect(received).toHaveLength(1); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "user_id") + ] + ).toEqual("float"); + return; + } + + expect(received).toHaveLength(2); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "user_id") + integrationCasedString(integrations[index], "user_id") ] ).toEqual("float"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("float"); }); @@ -707,22 +722,22 @@ describe("handle leading underscores in properties", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "_timestamp") + integrationCasedString(integrations[index], "_timestamp") ); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "__timestamp") + integrationCasedString(integrations[index], "__timestamp") ); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "__timestamp_new") + integrationCasedString(integrations[index], "__timestamp_new") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "_timestamp") + integrationCasedString(integrations[index], "_timestamp") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "__timestamp") + integrationCasedString(integrations[index], "__timestamp") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "__timestamp_new") + integrationCasedString(integrations[index], "__timestamp_new") ); }); }); @@ -736,22 +751,22 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); }); }); @@ -764,22 +779,22 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); }); }); @@ -792,17 +807,17 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("string"); }); @@ -818,28 +833,28 @@ describe("handle recordId from cloud sources", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toEqual("string"); expect( received[0].data[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toBe("42"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toBe(42); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); }); }); @@ -854,28 +869,28 @@ describe("handle recordId from cloud sources", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toEqual("string"); expect( received[0].data[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toBe("42"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toBe(42); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); }); }); @@ -915,10 +930,10 @@ describe("handle level three nested events from sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); }); }); @@ -943,10 +958,10 @@ describe("handle level three nested events from sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); }); }); @@ -956,7 +971,7 @@ describe("Handle no of columns in an event", () => { it("should throw an error if no of columns are more than 200", () => { const i = input("track"); transformers - .filter((transformer, index) => integration(index) !== "s3_datalake" && integration(index) !== "gcs_datalake" && integration(index) !== "azure_datalake") + .filter((transformer, index) => integrations[index] !== "s3_datalake" && integrations[index] !== "gcs_datalake" && integrations[index] !== "azure_datalake") .forEach((transformer, index) => { i.message.properties = largeNoOfColumnsevent; expect(() => transformer.process(i)).toThrow( @@ -984,13 +999,13 @@ describe("Add auto generated messageId for events missing it", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ); expect(received[0].data).toHaveProperty( - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toMatch(/auto-.*/); }); }); @@ -1007,10 +1022,10 @@ describe("Add receivedAt for events missing it", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "received_at") + integrationCasedString(integrations[index], "received_at") ); expect(received[0].data).toHaveProperty( - integrationCasedString(integration(index), "received_at") + integrationCasedString(integrations[index], "received_at") ); }); }); @@ -1023,22 +1038,22 @@ describe("Integration options", () => { const i = opInput("track"); transformers.forEach((transformer, index) => { const {jsonPaths} = i.destination.Config; - if (integration(index) === "postgres") { + if (integrations[index] === "postgres") { delete i.destination.Config.jsonPaths; } const received = transformer.process(i); i.destination.Config.jsonPaths = jsonPaths; - expect(received).toEqual(opOutput("track", integration(index))); + expect(received).toEqual(opOutput("track", integrations[index])); }); }); }); describe("users", () => { it("should skip users when skipUsersTable is set", () => { - const i = opInput("users"); + const i = opInput("identify"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toEqual(opOutput("users", integration(index))); + expect(received).toEqual(opOutput("identify", integrations[index])); }); }); }); @@ -1054,6 +1069,8 @@ describe("Integration options", () => { return _.cloneDeep(config.output.postgres); case "snowflake": return _.cloneDeep(config.output.snowflake); + case "snowpipe_streaming": + return _.cloneDeep(config.output.snowpipe_streaming); case "s3_datalake": case "gcs_datalake": case "azure_datalake": @@ -1093,18 +1110,18 @@ describe("Integration options", () => { for (const testCase of testCases) { transformers.forEach((transformer, index) => { - it(`new ${testCase.eventType} for ${integration(index)}`, () => { + it(`new ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(testCase.eventType, config, integration(index))); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) - it(`legacy ${testCase.eventType} for ${integration(index)}`, () => { + it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(testCase.eventType, config, integration(index))); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) }); } @@ -1242,7 +1259,10 @@ describe("Destination config", () => { transformers.forEach((transformer, index) => { const received = transformer.process(scenario.event); - expect(received).toHaveLength(scenario.expected.length); + const expectedLength = integrations[index] === "snowpipe_streaming" && scenario.event.message.type === "identify" + ? 1 + : scenario.expected.length; + expect(received).toHaveLength(expectedLength); for (const i in received) { const evt = received[i]; expect(evt.data.id ? evt.data.id : evt.data.ID).toEqual(scenario.expected[i].id); @@ -1285,7 +1305,9 @@ describe("Destination config", () => { } } const output = transformer.process(event); - const events = [output[0], output[1]]; // identifies and users event + const events = integrations[index] === "snowpipe_streaming" + ? [output[0]] + : [output[0], output[1]]; const traitsToCheck = { 'city': 'Disney', 'country': 'USA', @@ -1294,10 +1316,10 @@ describe("Destination config", () => { }; events.forEach(event => { Object.entries(traitsToCheck).forEach(([trait, value]) => { - expect(event.data[integrationCasedString(integration(index), trait)]).toEqual(value); - expect(event.data[integrationCasedString(integration(index), `context_traits_${trait}`)]).toEqual(value); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_traits_${trait}`)); + expect(event.data[integrationCasedString(integrations[index], trait)]).toEqual(value); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); }); }); }); @@ -1325,13 +1347,13 @@ describe("Destination config", () => { } } const output = transformer.process(event); - expect(output[0].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v_2'); - expect(output[0].data[integrationCasedString(integration(index), `context_attribute_v_3`)]).toEqual('some-value'); - expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v_3`)); - expect(output[1].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v_2'); - expect(output[1].data[integrationCasedString(integration(index), `context_attribute_v_3`)]).toEqual('some-value'); - expect(output[1].metadata.table).toEqual(integrationCasedString(integration(index), 'button_clicked_v_2')); - expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v_3`)); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v_2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); }); }); }); @@ -1364,7 +1386,10 @@ describe("Destination config", () => { } } const received = transformer.process(event); - const events = [received[0], received[1]]; // identifies and users event + const events = integrations[index] === "snowpipe_streaming" + ? [received[0]] + : [received[0], received[1]]; + // identifies and users event const traitsToCheck = { 'city': 'Disney', 'country': 'USA', @@ -1373,10 +1398,10 @@ describe("Destination config", () => { }; events.forEach(event => { Object.entries(traitsToCheck).forEach(([trait, value]) => { - expect(event.data).not.toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.data[integrationCasedString(integration(index), `context_traits_${trait}`)]).toEqual(value); - expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_traits_${trait}`)); + expect(event.data).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); }); }); }); @@ -1402,13 +1427,13 @@ describe("Destination config", () => { } } const output = transformer.process(event); - expect(output[0].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v2'); - expect(output[0].data[integrationCasedString(integration(index), `context_attribute_v3`)]).toEqual('some-value'); - expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v3`)); - expect(output[1].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v2'); - expect(output[1].data[integrationCasedString(integration(index), `context_attribute_v3`)]).toEqual('some-value'); - expect(output[1].metadata.table).toEqual(integrationCasedString(integration(index), 'button_clicked_v2')); - expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v3`)); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); }); }); }); @@ -1615,8 +1640,8 @@ describe("context traits", () => { expect(Object.keys(received[0].data).join()).not.toMatch(/context_traits/g); } for (const column of t.expectedColumns) { - expect(received[0].metadata.columns[integrationCasedString(integration(index), column)]).toEqual(t.expectedMetadata); - expect(received[0].data[integrationCasedString(integration(index), column)]).toEqual(t.expectedData); + expect(received[0].metadata.columns[integrationCasedString(integrations[index], column)]).toEqual(t.expectedMetadata); + expect(received[0].data[integrationCasedString(integrations[index], column)]).toEqual(t.expectedData); } }); } @@ -1708,8 +1733,8 @@ describe("group traits", () => { expect(Object.keys(received[0].data).join()).not.toMatch(/group_traits/g); } for (const column of t.expectedColumns) { - expect(received[0].metadata.columns[integrationCasedString(integration(index), column)]).toEqual(t.expectedMetadata); - expect(received[0].data[integrationCasedString(integration(index), column)]).toEqual(t.expectedData); + expect(received[0].metadata.columns[integrationCasedString(integrations[index], column)]).toEqual(t.expectedMetadata); + expect(received[0].data[integrationCasedString(integrations[index], column)]).toEqual(t.expectedData); } }); }); diff --git a/test/integrations/destinations/braze/processor/data.ts b/test/integrations/destinations/braze/processor/data.ts index 240206791e..9b6f6dac65 100644 --- a/test/integrations/destinations/braze/processor/data.ts +++ b/test/integrations/destinations/braze/processor/data.ts @@ -3160,7 +3160,7 @@ export const data = [ { subscription_group_id: '1234', subscription_state: 'subscribed', - external_id: ['Randomuser2222'], + external_ids: ['Randomuser2222'], phones: ['5055077683'], }, ], @@ -3281,7 +3281,7 @@ export const data = [ { subscription_group_id: '1234', subscription_state: 'unsubscribed', - external_id: ['Randomuser2222'], + external_ids: ['Randomuser2222'], emails: ['abc@test.com'], }, ], diff --git a/test/integrations/destinations/braze/router/data.ts b/test/integrations/destinations/braze/router/data.ts index 6803742e86..b788e22741 100644 --- a/test/integrations/destinations/braze/router/data.ts +++ b/test/integrations/destinations/braze/router/data.ts @@ -219,6 +219,34 @@ export const data = [ metadata: { jobId: 6, userId: 'u1' }, message: { type: 'alias', previousId: 'adsfsaf2', userId: 'dsafsdf2' }, }, + { + destination: { + Config: { + restApiKey: 'dummyApiKey', + prefixProperties: true, + useNativeSDK: false, + dataCenter: 'us-01', + enableSubscriptionGroupInGroupCall: true, + }, + DestinationDefinition: { + DisplayName: 'Braze', + ID: '1WhbSZ6uA3H5ChVifHpfL2H6sie', + Name: 'BRAZE', + }, + Enabled: true, + ID: '1WhcOCGgj9asZu850HvugU2C3Aq', + Name: 'Braze', + Transformations: [], + }, + metadata: { jobId: 7, userId: 'u1' }, + message: { + anonymousId: '56yrtsdfgbgxcb-22b4-401d-aae5-1b994be9afdf', + groupId: 'c90f0fd2-2a02-4f2f-bf07-7e7d2c2ed2b1', + traits: { phone: '5055077683', subscriptionState: 'subscribed' }, + userId: 'user12345', + type: 'group', + }, + }, ], destType: 'braze', }, @@ -299,13 +327,13 @@ export const data = [ JSON: { subscription_groups: [ { - external_id: ['user123'], + external_ids: ['user123', 'user12345'], phones: ['5055077683'], subscription_group_id: 'c90f0fd2-2a02-4f2f-bf07-7e7d2c2ed2b1', subscription_state: 'subscribed', }, { - external_id: ['user877'], + external_ids: ['user877'], phones: ['5055077683'], subscription_group_id: '58d0a278-b55b-4f10-b7d2-98d1c5dd4c30', subscription_state: 'subscribed', @@ -356,6 +384,7 @@ export const data = [ { jobId: 4, userId: 'u1' }, { jobId: 5, userId: 'u1' }, { jobId: 6, userId: 'u1' }, + { jobId: 7, userId: 'u1' }, ], batched: true, statusCode: 200, diff --git a/test/integrations/destinations/facebook_pixel/dataDelivery/business.ts b/test/integrations/destinations/facebook_pixel/dataDelivery/business.ts index 1b425ac5fa..2b4af61ac3 100644 --- a/test/integrations/destinations/facebook_pixel/dataDelivery/business.ts +++ b/test/integrations/destinations/facebook_pixel/dataDelivery/business.ts @@ -42,7 +42,7 @@ export const testScenariosForV1API: ProxyV1TestData[] = [ body: { output: { status: 400, - message: 'Invalid OAuth 2.0 access token', + message: 'The access token could not be decrypted', statTags: { ...statTags, errorCategory: 'dataValidation', @@ -51,7 +51,7 @@ export const testScenariosForV1API: ProxyV1TestData[] = [ }, response: [ { - error: 'Invalid OAuth 2.0 access token', + error: 'The access token could not be decrypted', statusCode: 400, metadata: generateMetadata(1), }, diff --git a/test/integrations/destinations/facebook_pixel/dataDelivery/data.ts b/test/integrations/destinations/facebook_pixel/dataDelivery/data.ts index 3366e62c5a..0e004c1183 100644 --- a/test/integrations/destinations/facebook_pixel/dataDelivery/data.ts +++ b/test/integrations/destinations/facebook_pixel/dataDelivery/data.ts @@ -45,12 +45,13 @@ export const v0TestData = [ body: { output: { status: 400, - message: 'Invalid OAuth 2.0 access token', + message: 'The access token could not be decrypted', destinationResponse: { error: { code: 190, fbtrace_id: 'fbpixel_trace_id', message: 'The access token could not be decrypted', + error_user_msg: 'The access token could not be decrypted', type: 'OAuthException', }, status: 500, @@ -202,7 +203,7 @@ export const v0TestData = [ body: { output: { status: 429, - message: 'API User Too Many Calls', + message: 'User request limit reached', destinationResponse: { error: { message: 'User request limit reached', @@ -414,7 +415,7 @@ export const v0TestData = [ body: { output: { status: 400, - message: 'Capability or permissions issue.', + message: 'Some error in permission', destinationResponse: { error: { message: 'Some error in permission', @@ -466,7 +467,8 @@ export const v0TestData = [ body: { output: { status: 500, - message: 'Unhandled random error', + message: + '{"message":"Unhandled random error","type":"RandomException","code":5,"error_subcode":12,"fbtrace_id":"facebook_px_trace_id_10"}', destinationResponse: { error: { message: 'Unhandled random error', @@ -488,6 +490,66 @@ export const v0TestData = [ }, }, + { + name: 'facebook_pixel', + description: 'Test 10: should handle error with code: 100 & subcode: 2804009', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + body: { + XML: {}, + FORM: testFormData, + JSON: {}, + JSON_ARRAY: {}, + }, + type: 'REST', + files: {}, + method: 'POST', + userId: '', + headers: {}, + version: '1', + endpoint: `https://graph.facebook.com/${VERSION}/12345678912804009/events?access_token=2804009_valid_access_token`, + params: { + destination: 'facebook_pixel', + }, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 400, + body: { + output: { + status: 400, + message: + 'Your purchase event doesn’t include a value parameter. Enter a value. For example: 1.99', + destinationResponse: { + error: { + message: 'Invalid parameter', + type: 'OAuthException', + code: 100, + error_user_msg: + 'Your purchase event doesn’t include a value parameter. Enter a value. For example: 1.99', + error_user_title: 'Missing Value for Purchase Event', + error_subcode: 2804009, + is_transient: false, + fbtrace_id: 'AP4G-xxxxxxxx', + }, + status: 400, + }, + statTags: { + ...statTags, + errorType: 'aborted', + }, + }, + }, + }, + }, + }, { name: 'facebook_pixel', description: 'Test 9: should handle error with code: 21009', diff --git a/test/integrations/destinations/facebook_pixel/dataDelivery/oauth.ts b/test/integrations/destinations/facebook_pixel/dataDelivery/oauth.ts index c6d938c627..bb4fd582f0 100644 --- a/test/integrations/destinations/facebook_pixel/dataDelivery/oauth.ts +++ b/test/integrations/destinations/facebook_pixel/dataDelivery/oauth.ts @@ -28,11 +28,11 @@ export const oauthScenariosV1: ProxyV1TestData[] = [ body: { output: { status: 400, - message: 'Capability or permissions issue.', + message: 'Some error in permission', statTags, response: [ { - error: 'Capability or permissions issue.', + error: 'Some error in permission', statusCode: 400, metadata: generateMetadata(1), }, diff --git a/test/integrations/destinations/facebook_pixel/dataDelivery/other.ts b/test/integrations/destinations/facebook_pixel/dataDelivery/other.ts index e25cc8e07c..154c6f75ad 100644 --- a/test/integrations/destinations/facebook_pixel/dataDelivery/other.ts +++ b/test/integrations/destinations/facebook_pixel/dataDelivery/other.ts @@ -31,14 +31,14 @@ export const otherScenariosV1: ProxyV1TestData[] = [ body: { output: { status: 429, - message: 'API User Too Many Calls', + message: 'User request limit reached', statTags: { ...statTags, errorType: 'throttled', }, response: [ { - error: 'API User Too Many Calls', + error: 'User request limit reached', statusCode: 429, metadata: generateMetadata(1), }, @@ -72,7 +72,8 @@ export const otherScenariosV1: ProxyV1TestData[] = [ body: { output: { status: 500, - message: 'Unhandled random error', + message: + '{"message":"Unhandled random error","type":"RandomException","code":5,"error_subcode":12,"fbtrace_id":"facebook_px_trace_id_10"}', statTags: { ...statTags, errorType: 'retryable', @@ -80,7 +81,8 @@ export const otherScenariosV1: ProxyV1TestData[] = [ }, response: [ { - error: 'Unhandled random error', + error: + '{"message":"Unhandled random error","type":"RandomException","code":5,"error_subcode":12,"fbtrace_id":"facebook_px_trace_id_10"}', statusCode: 500, metadata: generateMetadata(1), }, diff --git a/test/integrations/destinations/facebook_pixel/network.ts b/test/integrations/destinations/facebook_pixel/network.ts index 411d36cf19..a6ff50623b 100644 --- a/test/integrations/destinations/facebook_pixel/network.ts +++ b/test/integrations/destinations/facebook_pixel/network.ts @@ -15,6 +15,7 @@ export const networkCallsData = [ data: { error: { message: 'The access token could not be decrypted', + error_user_msg: 'The access token could not be decrypted', type: 'OAuthException', code: 190, fbtrace_id: 'fbpixel_trace_id', @@ -208,4 +209,29 @@ export const networkCallsData = [ statusText: 'OK', }, }, + { + httpReq: { + url: `https://graph.facebook.com/${VERSION}/12345678912804009/events?access_token=2804009_valid_access_token`, + data: getFormData(testFormData).toString(), + params: { destination: 'facebook_pixel' }, + headers: { 'User-Agent': 'RudderLabs' }, + method: 'POST', + }, + httpRes: { + status: 400, + data: { + error: { + code: 100, + error_subcode: 2804009, + error_user_msg: + 'Your purchase event doesn’t include a value parameter. Enter a value. For example: 1.99', + error_user_title: 'Missing Value for Purchase Event', + fbtrace_id: 'AP4G-xxxxxxxx', + is_transient: false, + message: 'Invalid parameter', + type: 'OAuthException', + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/fb/dataDelivery/data.ts b/test/integrations/destinations/fb/dataDelivery/data.ts index 9ee19af265..dfa5dbc65e 100644 --- a/test/integrations/destinations/fb/dataDelivery/data.ts +++ b/test/integrations/destinations/fb/dataDelivery/data.ts @@ -59,6 +59,7 @@ export const existingTestData = [ message: 'Invalid OAuth 2.0 access token', destinationResponse: { error: { + error_user_msg: 'Invalid OAuth 2.0 access token', code: 190, fbtrace_id: 'fbpixel_trace_id', message: 'The access token could not be decrypted', @@ -274,7 +275,7 @@ export const existingTestData = [ body: { output: { status: 429, - message: 'API User Too Many Calls', + message: 'User request limit reached', destinationResponse: { error: { message: 'User request limit reached', diff --git a/test/integrations/destinations/fb/dataDelivery/other.ts b/test/integrations/destinations/fb/dataDelivery/other.ts index 9ac3f14fb5..36d3e45f53 100644 --- a/test/integrations/destinations/fb/dataDelivery/other.ts +++ b/test/integrations/destinations/fb/dataDelivery/other.ts @@ -31,14 +31,14 @@ export const otherScenariosV1: ProxyV1TestData[] = [ body: { output: { status: 429, - message: 'API User Too Many Calls', + message: 'User request limit reached', statTags: { ...statTags, errorType: 'throttled', }, response: [ { - error: 'API User Too Many Calls', + error: 'User request limit reached', statusCode: 429, metadata: generateMetadata(1), }, diff --git a/test/integrations/destinations/fb/network.ts b/test/integrations/destinations/fb/network.ts index 31bbaf0b6e..1aed48136c 100644 --- a/test/integrations/destinations/fb/network.ts +++ b/test/integrations/destinations/fb/network.ts @@ -30,6 +30,7 @@ export const networkCallsData = [ data: { error: { message: 'The access token could not be decrypted', + error_user_msg: 'Invalid OAuth 2.0 access token', type: 'OAuthException', code: 190, fbtrace_id: 'fbpixel_trace_id', diff --git a/test/integrations/destinations/fb_custom_audience/dataDelivery/data.ts b/test/integrations/destinations/fb_custom_audience/dataDelivery/data.ts index b41c656d9f..cd440aaa37 100644 --- a/test/integrations/destinations/fb_custom_audience/dataDelivery/data.ts +++ b/test/integrations/destinations/fb_custom_audience/dataDelivery/data.ts @@ -623,6 +623,8 @@ export const existingTestData = [ fbtrace_id: 'A3b8C6PpI-kdIOwPwV4PANi', message: 'Error validating access token: Session has expired on Tuesday, 01-Aug-23 10:12:14 PDT. The current time is Sunday, 28-Jan-24 16:01:17 PST.', + error_user_msg: + 'Error validating access token: Session has expired on Tuesday, 01-Aug-23 10:12:14 PDT. The current time is Sunday, 28-Jan-24 16:01:17 PST.', type: 'OAuthException', }, status: 400, diff --git a/test/integrations/destinations/fb_custom_audience/network.ts b/test/integrations/destinations/fb_custom_audience/network.ts index 369c27afa9..ba14d537ad 100644 --- a/test/integrations/destinations/fb_custom_audience/network.ts +++ b/test/integrations/destinations/fb_custom_audience/network.ts @@ -514,6 +514,8 @@ export const networkCallsData = [ error: { message: 'Error validating access token: Session has expired on Tuesday, 01-Aug-23 10:12:14 PDT. The current time is Sunday, 28-Jan-24 16:01:17 PST.', + error_user_msg: + 'Error validating access token: Session has expired on Tuesday, 01-Aug-23 10:12:14 PDT. The current time is Sunday, 28-Jan-24 16:01:17 PST.', type: 'OAuthException', code: 190, error_subcode: 463, diff --git a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts index ab3e19dc2f..3ecae2b92d 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts @@ -231,8 +231,6 @@ export const data = [ JSON: { conversions: [ { - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -528,8 +526,6 @@ export const data = [ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', }, - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -821,8 +817,6 @@ export const data = [ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', }, - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -1911,7 +1905,6 @@ export const data = [ userId: '12345', properties: { gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', conversionCustomVariable: 'conversionCustomVariable', @@ -1935,7 +1928,6 @@ export const data = [ ], userIdentifierSource: 'FIRST_PARTY', conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: '1', currency: 'GBP', @@ -2052,7 +2044,6 @@ export const data = [ ], properties: { gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', conversionCustomVariable: 'conversionCustomVariable', @@ -2076,7 +2067,6 @@ export const data = [ ], userIdentifierSource: 'FIRST_PARTY', conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: '1', currency: 'GBP', @@ -2093,7 +2083,6 @@ export const data = [ adUserData: 'UNSPECIFIED', }, gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -2110,15 +2099,7 @@ export const data = [ }, ], }, - userIdentifiers: [ - { - userIdentifierSource: 'FIRST_PARTY', - hashedPhoneNumber: - '04e1dabb7c1348b72bfa87da179c9697c69af74827649266a5da8cdbb367abcd', - }, - ], conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: 1, currencyCode: 'GBP', @@ -2169,7 +2150,6 @@ export const data = [ ], properties: { gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', conversionCustomVariable: 'conversionCustomVariable', @@ -2193,7 +2173,6 @@ export const data = [ ], userIdentifierSource: 'FIRST_PARTY', conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: '1', currency: 'GBP', @@ -3751,20 +3730,73 @@ export const data = [ metadata: { secret: { access_token: 'abcd1234', - refresh_token: 'efgh5678', developer_token: 'ijkl91011', + refresh_token: 'efgh5678', }, }, - statusCode: 400, - error: 'Either of email or phone is required for user identifier', - statTags: { - errorCategory: 'dataValidation', - errorType: 'instrumentation', - destType: 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS', - module: 'destination', - implementation: 'native', - feature: 'processor', + output: { + body: { + FORM: {}, + JSON: { + conversions: [ + { + cartData: { + items: [ + { + productId: '123445', + quantity: 123, + }, + ], + }, + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + conversionDateTime: '2022-01-01 12:32:45-08:00', + conversionEnvironment: 'WEB', + gclid: 'gclid', + }, + ], + partialFailure: true, + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + files: {}, + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + 'login-customer-id': '8617859087', + }, + method: 'POST', + params: { + customVariables: [ + { + from: 'value', + to: 'revenue', + }, + { + from: 'total', + to: 'cost', + }, + ], + customerId: '9625812972', + event: 'Sign-up - click', + properties: { + conversionDateTime: '2022-01-01 12:32:45-08:00', + gclid: 'gclid', + product_id: '123445', + quantity: 123, + }, + }, + type: 'REST', + userId: '', + version: '1', }, + statusCode: 200, }, ], }, @@ -5704,4 +5736,197 @@ export const data = [ }, mockFns: timestampMock, }, + { + name: 'google_adwords_offline_conversions', + description: 'Test 28 : when both gbraid and wbraid are available', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + id: '0572f78fa49c648e', + name: 'generic_x86_arm', + type: 'Android', + model: 'AOSP on IA Emulator', + manufacturer: 'Google', + adTrackingEnabled: true, + advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8', + }, + traits: { + phone: 'alex@example.com', + firstName: 'John', + lastName: 'Gomes', + city: 'London', + state: 'England', + countryCode: 'GB', + postalCode: 'EC3M', + streetAddress: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + }, + event: 'Promotion Clicked', + type: 'track', + messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', + originalTimestamp: '2019-10-14T11:15:18.299Z', + anonymousId: '00000000000000000000000000', + userId: '12345', + properties: { + gbraid: 'gbraid', + wbraid: 'wbraid', + externalAttributionCredit: 10, + externalAttributionModel: 'externalAttributionModel', + conversionCustomVariable: 'conversionCustomVariable', + value: 'value', + merchantId: '9876merchantId', + feedCountryCode: 'feedCountryCode', + feedLanguageCode: 'feedLanguageCode', + localTransactionCost: 20, + products: [ + { + product_id: '507f1f77bcf86cd799439011', + quantity: '2', + price: '50', + sku: '45790-32', + name: 'Monopoly: 3rd Edition', + position: '1', + category: 'cars', + url: 'https://www.example.com/product/path', + image_url: 'https://www.example.com/product/path.jpg', + }, + ], + userIdentifierSource: 'FIRST_PARTY', + conversionEnvironment: 'WEB', + conversionValue: '1', + currency: 'GBP', + orderId: 'PL-123QR', + }, + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + customerId: '962-581-2972', + subAccount: false, + eventsToOfflineConversionsTypeMapping: [ + { + from: 'Sign up completed', + to: 'click', + }, + { + from: 'Download', + to: 'call', + }, + { + from: 'Promotion Clicked', + to: 'click', + }, + { + from: 'Product Searched', + to: 'call', + }, + ], + eventsToConversionsNamesMapping: [ + { + from: 'Sign up completed', + to: 'Sign-up - click', + }, + { + from: 'Download', + to: 'Page view', + }, + { + from: 'Promotion Clicked', + to: 'Sign-up - click', + }, + { + from: 'Product Searched', + to: 'search', + }, + ], + customVariables: [ + { + from: 'value', + to: 'revenue', + }, + { + from: 'total', + to: 'cost', + }, + ], + UserIdentifierSource: 'THIRD_PARTY', + conversionEnvironment: 'WEB', + hashUserIdentifier: true, + defaultUserIdentifier: 'email', + validateOnly: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: "You can't use both wbraid and gbraid.", + metadata: { + secret: { + access_token: 'abcd1234', + developer_token: 'ijkl91011', + refresh_token: 'efgh5678', + }, + }, + statTags: { + destType: 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + mockFns: timestampMock, + }, ]; diff --git a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts index 9d1ba220c8..82c8e9b3ff 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts @@ -738,8 +738,6 @@ export const data = [ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', }, - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', diff --git a/test/integrations/sources/shopify/constants.ts b/test/integrations/sources/shopify/constants.ts index f9df305841..af53a3180e 100644 --- a/test/integrations/sources/shopify/constants.ts +++ b/test/integrations/sources/shopify/constants.ts @@ -83,6 +83,21 @@ export const dummyContext = { }, }; +export const note_attributes = [ + { + name: 'cartId', + value: '9c623f099fc8819aa4d6a958b65dfe7d', + }, + { + name: 'cartToken', + value: 'Z2NwLXVzLWVhc3QxOjAxSkQzNUFXVEI4VkVUNUpTTk1LSzBCMzlF', + }, + { + name: 'rudderAnonymousId', + value: '50ead33e-d763-4854-b0ab-765859ef05cb', + }, +]; + export const responseDummyContext = { document: { location: { diff --git a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts index ade496efb7..a154ccb890 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts @@ -1,7 +1,6 @@ // This file contains the test scenarios for the server-side events from the Shopify GraphQL API for // the v1 transformation flow -import { mockFns } from '../mocks'; -import { dummySourceConfig } from '../constants'; +import { dummySourceConfig, note_attributes } from '../constants'; export const checkoutEventsTestScenarios = [ { @@ -27,7 +26,7 @@ export const checkoutEventsTestScenarios = [ updated_at: '2024-11-05T21:22:02-05:00', landing_site: '/', note: '', - note_attributes: [], + note_attributes, referring_site: '', shipping_lines: [], shipping_address: [], @@ -145,7 +144,7 @@ export const checkoutEventsTestScenarios = [ updated_at: '2024-11-05T21:22:02-05:00', landing_site: '/', note: '', - note_attributes: [], + note_attributes, referring_site: '', shipping_lines: [], shipping_address: [], @@ -237,7 +236,7 @@ export const checkoutEventsTestScenarios = [ traits: { shippingAddress: [], }, - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', }, ], }, @@ -269,7 +268,7 @@ export const checkoutEventsTestScenarios = [ created_at: '2024-09-16T03:50:1500:00', updated_at: '2024-09-17T03:29:02-04:00', note: '', - note_attributes: [], + note_attributes, shipping_address: { first_name: 'testuser', address1: 'oakwood bridge', @@ -396,7 +395,7 @@ export const checkoutEventsTestScenarios = [ output: { batch: [ { - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', context: { cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', integration: { @@ -415,7 +414,7 @@ export const checkoutEventsTestScenarios = [ created_at: '2024-09-16T03:50:1500:00', updated_at: '2024-09-17T03:29:02-04:00', note: '', - note_attributes: [], + note_attributes, shipping_address: { first_name: 'testuser', address1: 'oakwood bridge', @@ -680,7 +679,7 @@ export const checkoutEventsTestScenarios = [ merchant_of_record_app_id: null, name: '#1017', note: null, - note_attributes: [], + note_attributes, number: 17, order_number: 1017, order_status_url: @@ -988,7 +987,7 @@ export const checkoutEventsTestScenarios = [ merchant_of_record_app_id: null, name: '#1017', note: null, - note_attributes: [], + note_attributes, number: 17, order_number: 1017, order_status_url: @@ -1289,7 +1288,7 @@ export const checkoutEventsTestScenarios = [ }, }, timestamp: '2024-11-06T02:54:50.000Z', - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', }, ], }, @@ -1326,6 +1325,7 @@ export const checkoutEventsTestScenarios = [ current_total_tax: '0.00', email: 'henry@wfls.com', name: '#1017', + note_attributes, order_number: 1017, order_status_url: 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', @@ -1486,6 +1486,7 @@ export const checkoutEventsTestScenarios = [ current_total_tax: '0.00', email: 'henry@wfls.com', name: '#1017', + note_attributes, order_number: 1017, order_status_url: 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', @@ -1675,7 +1676,7 @@ export const checkoutEventsTestScenarios = [ }, }, timestamp: '2024-11-06T02:54:50.000Z', - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', }, ], }, @@ -1684,4 +1685,4 @@ export const checkoutEventsTestScenarios = [ }, }, }, -].map((d1) => ({ ...d1, mockFns })); +]; diff --git a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts index f04fd7e08e..d68d0a8f59 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts @@ -1,7 +1,7 @@ // This file contains the test scenarios for the server-side events from the Shopify GraphQL API for // the v1 transformation flow import { mockFns } from '../mocks'; -import { dummySourceConfig } from '../constants'; +import { dummySourceConfig, note_attributes } from '../constants'; export const genericTrackTestScenarios = [ { @@ -24,6 +24,7 @@ export const genericTrackTestScenarios = [ token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', line_items: [], note: '', + note_attributes, updated_at: '2024-09-17T08:15:13.280Z', created_at: '2024-09-16T03:50:15.478Z', }, @@ -43,7 +44,7 @@ export const genericTrackTestScenarios = [ output: { batch: [ { - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', context: { integration: { name: 'SHOPIFY', @@ -58,6 +59,7 @@ export const genericTrackTestScenarios = [ id: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', line_items: [], note: '', + note_attributes, token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', updated_at: '2024-09-17T08:15:13.280Z', }, @@ -149,7 +151,7 @@ export const genericTrackTestScenarios = [ merchant_of_record_app_id: null, name: '#1017', note: null, - note_attributes: [], + note_attributes, order_number: 1017, original_total_additional_fees_set: null, original_total_duties_set: null, @@ -363,7 +365,7 @@ export const genericTrackTestScenarios = [ merchant_of_record_app_id: null, name: '#1017', note: null, - note_attributes: [], + note_attributes, order_number: 1017, original_total_additional_fees_set: null, original_total_duties_set: null, @@ -545,7 +547,7 @@ export const genericTrackTestScenarios = [ traits: { email: 'henry@wfls.com', }, - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', }, ], },