From fb1595f9fdf10a296bedae797967c97289947885 Mon Sep 17 00:00:00 2001 From: Gauravudia Date: Thu, 28 Sep 2023 11:34:05 +0530 Subject: [PATCH] feat: mixpanel gzip support for import endpoint --- src/v0/destinations/mp/util.js | 21 ++++-- src/v0/destinations/mp/util.test.js | 79 ++++++++++++++++++++++- test/__tests__/data/mp_router_output.json | 30 +++++---- 3 files changed, 113 insertions(+), 17 deletions(-) diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index c5fc7e04be..baabb4ee01 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -1,5 +1,6 @@ const set = require('set-value'); const get = require('get-value'); +const zlib = require('zlib'); const { isDefined, constructPayload, @@ -212,11 +213,22 @@ const groupEventsByEndpoint = (events) => { const generateBatchedPayloadForArray = (events) => { const { batchedRequest } = defaultBatchRequestConfig(); + const firstEvent = events[0]; + batchedRequest.endpoint = firstEvent.endpoint; + batchedRequest.headers = firstEvent.headers; + batchedRequest.params = firstEvent.params; + const batchResponseList = events.flatMap((event) => JSON.parse(event.body.JSON_ARRAY.batch)); - batchedRequest.body.JSON_ARRAY = { batch: JSON.stringify(batchResponseList) }; - batchedRequest.endpoint = events[0].endpoint; - batchedRequest.headers = events[0].headers; - batchedRequest.params = events[0].params; + // Gzipping the payload for /import endpoint + if (firstEvent.endpoint.includes('import')) { + batchedRequest.body.GZIP = { + payload: zlib.gzipSync(JSON.stringify(batchResponseList)).toString('base64'), + }; + batchedRequest.headers['Content-Encoding'] = 'gzip'; + } else { + batchedRequest.body.JSON_ARRAY = { batch: JSON.stringify(batchResponseList) }; + } + return batchedRequest; }; @@ -284,6 +296,7 @@ module.exports = { createIdentifyResponse, isImportAuthCredentialsAvailable, groupEventsByEndpoint, + generateBatchedPayloadForArray, batchEvents, combineBatchRequestsWithSameJobIds, }; diff --git a/src/v0/destinations/mp/util.test.js b/src/v0/destinations/mp/util.test.js index 43594fba6d..cb209901be 100644 --- a/src/v0/destinations/mp/util.test.js +++ b/src/v0/destinations/mp/util.test.js @@ -1,8 +1,8 @@ const { combineBatchRequestsWithSameJobIds, - groupEventsByType, groupEventsByEndpoint, batchEvents, + generateBatchedPayloadForArray, } = require('./util'); const destinationMock = { @@ -488,4 +488,81 @@ describe('Mixpanel utils test', () => { expect(combineBatchRequestsWithSameJobIds(input)).toEqual(expectedOutput); }); }); + + describe('Unit test cases for generateBatchedPayloadForArray', () => { + it('should generate a batched payload with GZIP payload for /import endpoint when given an array of events', () => { + const events = [ + { + body: { JSON_ARRAY: { batch: '[{"event": "event1"}]' } }, + endpoint: '/import', + headers: { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' }, + params: {}, + }, + { + body: { JSON_ARRAY: { batch: '[{"event": "event2"}]' } }, + endpoint: '/import', + headers: { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' }, + params: {}, + }, + ]; + const expectedBatchedRequest = { + body: { + FORM: {}, + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + GZIP: { + payload: 'H4sIAAAAAAAAE4uuVkotS80rUbKC0IZKtTpoQkZKtbEAcv1DfCcAAAA=', + }, + }, + endpoint: '/import', + files: {}, + headers: { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }; + + const result = generateBatchedPayloadForArray(events); + + expect(result).toEqual(expectedBatchedRequest); + }); + + it('should generate a batched payload with JSON_ARRAY body when given an array of events', () => { + const events = [ + { + body: { JSON_ARRAY: { batch: '[{"event": "event1"}]' } }, + endpoint: '/endpoint', + headers: { 'Content-Type': 'application/json' }, + params: {}, + }, + { + body: { JSON_ARRAY: { batch: '[{"event": "event2"}]' } }, + endpoint: '/endpoint', + headers: { 'Content-Type': 'application/json' }, + params: {}, + }, + ]; + const expectedBatchedRequest = { + body: { + FORM: {}, + JSON: {}, + JSON_ARRAY: { batch: '[{"event":"event1"},{"event":"event2"}]' }, + XML: {}, + }, + endpoint: '/endpoint', + files: {}, + headers: { 'Content-Type': 'application/json' }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }; + + const result = generateBatchedPayloadForArray(events); + + expect(result).toEqual(expectedBatchedRequest); + }); + }); }); diff --git a/test/__tests__/data/mp_router_output.json b/test/__tests__/data/mp_router_output.json index d1af209e8b..df4057a414 100644 --- a/test/__tests__/data/mp_router_output.json +++ b/test/__tests__/data/mp_router_output.json @@ -7,6 +7,7 @@ "endpoint": "https://api.mixpanel.com/import/", "headers": { "Content-Type": "application/json", + "Content-Encoding": "gzip", "Authorization": "Basic dGVzdF9hcGlfc2VjcmV0Og==" }, "params": { @@ -14,11 +15,12 @@ }, "body": { "JSON": {}, - "JSON_ARRAY": { - "batch": "[{\"event\":\"Loaded a Page\",\"properties\":{\"ip\":\"0.0.0.0\",\"$user_id\":\"hjikl\",\"$current_url\":\"https://docs.rudderstack.com/destinations/mixpanel\",\"$screen_dpi\":2,\"mp_lib\":\"RudderLabs JavaScript SDK\",\"$app_build_number\":\"1.0.0\",\"$app_version_string\":\"1.0.5\",\"$insert_id\":\"dd266c67-9199-4a52-ba32-f46ddde67312\",\"token\":\"test_api_token\",\"distinct_id\":\"hjikl\",\"time\":1688624942,\"name\":\"Contact Us\",\"$browser\":\"Chrome\",\"$browser_version\":\"79.0.3945.117\"}}]" - }, + "JSON_ARRAY": {}, "XML": {}, - "FORM": {} + "FORM": {}, + "GZIP": { + "payload": "H4sIAAAAAAAAE1VQQU7DMBD8SmRxbBKSpkmTazlBD4iKE0KWYy/t0sS2bKcgVf0765RKoL3NzM7O7NuZwQl0YB3bGqFAJSJ5FntgC2adseACgmfdmaElyX02D5F3kwfHURF4+MTjECE5OUdWfHJDhEOwvstzZaTP3KQUOB+EPGbSjLkCH1CLgEb7fMRvKzTMHl46AM2VRdaVCzZaPmBPbi+zwVb0PnkUJ7GTDm1Idg9PcUlYy/sJB8X1NPbgSF/cckbuRKfpEvfBod7/sqvIoqYa4dpDqbKuZd2kbdG2aSVWZdqLZZl+VLWi43WzLEraCeYImuSBKnBhkV+BBVMYO8nw/ysBR2BdUa/XdVm1FXXSIiJsYzS9IySvPgbpnfnyc/LNwZkR/mC3+MQ1LQVfttUqK4qGXS7vP4+w2Dm/AQAA" + } }, "files": {} }, @@ -76,6 +78,7 @@ "endpoint": "https://api.mixpanel.com/import/", "headers": { "Content-Type": "application/json", + "Content-Encoding": "gzip", "Authorization": "Basic dGVzdF9hcGlfc2VjcmV0Og==" }, "params": { @@ -83,11 +86,12 @@ }, "body": { "JSON": {}, - "JSON_ARRAY": { - "batch": "[{\"event\":\"Product Viewed\",\"properties\":{\"name\":\"T-Shirt\",\"revenue\":18.9,\"$user_id\":\"userId01\",\"$os\":\"iOS\",\"$screen_height\":1794,\"$screen_width\":1080,\"$screen_dpi\":420,\"$carrier\":\"Android\",\"$os_version\":\"8.1.0\",\"$device\":\"generic_x86\",\"$manufacturer\":\"Google\",\"$model\":\"Android SDK built for x86\",\"mp_device_model\":\"Android SDK built for x86\",\"$wifi\":true,\"$bluetooth_enabled\":false,\"mp_lib\":\"com.rudderstack.android.sdk.core\",\"$app_build_number\":\"1\",\"$app_version_string\":\"1.0\",\"$insert_id\":\"id2\",\"token\":\"test_api_token\",\"distinct_id\":\"userId01\",\"time\":1688624942,\"$device_id\":\"anonId01\"}}]" - }, + "JSON_ARRAY": {}, "XML": {}, - "FORM": {} + "FORM": {}, + "GZIP": { + "payload": "H4sIAAAAAAAAE42RQUsDMRCF/8oSeqxLt5S67U0QRDwoVLyIhGwy7Q7dTZbJpBXE/+5kV+3Bi7fkmzcvkzevHwpO4Flt1RMFlywXLwhncGquBgoDECNEtf1Q3vQgquerXYvEUqbcmIRVdbmZq1mKQBqdaPLp3i0qEc2CNCt83OVztATgdQt4aOXF6nqzutAzOm4FLurFBboB1Xa1zMQaIgQStxvvKKCb3PUJKGLwwuuyKheZOjihzcMewAOh1e/1OvPe+LQ3lhONPnchHDoYC8FBd3EudrcPRZOw42IfqJi6+0FPvvo/6tkZ9zI6UwK5NF0CDoFbDd40naS73ZsuwujaYSNmNvQlJefkN2zssTSTeRndsbSBxjHNMOj8kNM+9c34h+qHf8egIxP6Q65MWaCXZfC0F3RLQRyOkONiiKzNgHoCc+UwMnrLf5bImFdfret6vVxtVsvfiCel8cGPys/Pty8LjzqIUAIAAA==" + } }, "files": {} } @@ -145,6 +149,7 @@ "endpoint": "https://api.mixpanel.com/import/", "headers": { "Content-Type": "application/json", + "Content-Encoding": "gzip", "Authorization": "Basic dGVzdF9hcGlfc2VjcmV0Og==" }, "params": { @@ -152,11 +157,12 @@ }, "body": { "JSON": {}, - "JSON_ARRAY": { - "batch": "[{\"event\":\"$merge\",\"properties\":{\"$distinct_ids\":[\"test_user_id\",\"5094f5704b9cf2b3\"],\"token\":\"test_api_token\"}}]" - }, + "JSON_ARRAY": {}, "XML": {}, - "FORM": {} + "FORM": {}, + "GZIP": { + "payload": "H4sIAAAAAAAAEyXMSQqAMBBE0bsULl2IA6JXEQkOpTRiDEnrRry7QbePz+9u8KJVtEh2+pVI4fzh6FUY0N5IZgkqdlIjc4QOyqDmDPQRYl1lTblUdVaOzbTkY4E+hR4bbVx+6eDE/PA8/QuxJ+OqcAAAAA==" + } }, "files": {} }