Skip to content

Commit

Permalink
feat: mixpanel gzip support for import endpoint
Browse files Browse the repository at this point in the history
  • Loading branch information
@Gauravudia
Gauravudia committed Sep 28, 2023
1 parent 10a13bc commit fb1595f
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 17 deletions.
21 changes: 17 additions & 4 deletions src/v0/destinations/mp/util.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const set = require('set-value');
const get = require('get-value');
const zlib = require('zlib');
const {
isDefined,
constructPayload,
Expand Down Expand Up @@ -212,11 +213,22 @@ const groupEventsByEndpoint = (events) => {

const generateBatchedPayloadForArray = (events) => {
const { batchedRequest } = defaultBatchRequestConfig();
const firstEvent = events[0];
batchedRequest.endpoint = firstEvent.endpoint;
batchedRequest.headers = firstEvent.headers;
batchedRequest.params = firstEvent.params;

const batchResponseList = events.flatMap((event) => JSON.parse(event.body.JSON_ARRAY.batch));
batchedRequest.body.JSON_ARRAY = { batch: JSON.stringify(batchResponseList) };
batchedRequest.endpoint = events[0].endpoint;
batchedRequest.headers = events[0].headers;
batchedRequest.params = events[0].params;
// Gzipping the payload for /import endpoint
if (firstEvent.endpoint.includes('import')) {
batchedRequest.body.GZIP = {
payload: zlib.gzipSync(JSON.stringify(batchResponseList)).toString('base64'),
};
batchedRequest.headers['Content-Encoding'] = 'gzip';
} else {
batchedRequest.body.JSON_ARRAY = { batch: JSON.stringify(batchResponseList) };
}

return batchedRequest;
};

Expand Down Expand Up @@ -284,6 +296,7 @@ module.exports = {
createIdentifyResponse,
isImportAuthCredentialsAvailable,
groupEventsByEndpoint,
generateBatchedPayloadForArray,
batchEvents,
combineBatchRequestsWithSameJobIds,
};
79 changes: 78 additions & 1 deletion src/v0/destinations/mp/util.test.js
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
const {
combineBatchRequestsWithSameJobIds,
groupEventsByType,
groupEventsByEndpoint,
batchEvents,
generateBatchedPayloadForArray,
} = require('./util');

const destinationMock = {
Expand Down Expand Up @@ -488,4 +488,81 @@ describe('Mixpanel utils test', () => {
expect(combineBatchRequestsWithSameJobIds(input)).toEqual(expectedOutput);
});
});

describe('Unit test cases for generateBatchedPayloadForArray', () => {
it('should generate a batched payload with GZIP payload for /import endpoint when given an array of events', () => {
const events = [
{
body: { JSON_ARRAY: { batch: '[{"event": "event1"}]' } },
endpoint: '/import',
headers: { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' },
params: {},
},
{
body: { JSON_ARRAY: { batch: '[{"event": "event2"}]' } },
endpoint: '/import',
headers: { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' },
params: {},
},
];
const expectedBatchedRequest = {
body: {
FORM: {},
JSON: {},
JSON_ARRAY: {},
XML: {},
GZIP: {
payload: 'H4sIAAAAAAAAE4uuVkotS80rUbKC0IZKtTpoQkZKtbEAcv1DfCcAAAA=',
},
},
endpoint: '/import',
files: {},
headers: { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' },
method: 'POST',
params: {},
type: 'REST',
version: '1',
};

const result = generateBatchedPayloadForArray(events);

expect(result).toEqual(expectedBatchedRequest);
});

it('should generate a batched payload with JSON_ARRAY body when given an array of events', () => {
const events = [
{
body: { JSON_ARRAY: { batch: '[{"event": "event1"}]' } },
endpoint: '/endpoint',
headers: { 'Content-Type': 'application/json' },
params: {},
},
{
body: { JSON_ARRAY: { batch: '[{"event": "event2"}]' } },
endpoint: '/endpoint',
headers: { 'Content-Type': 'application/json' },
params: {},
},
];
const expectedBatchedRequest = {
body: {
FORM: {},
JSON: {},
JSON_ARRAY: { batch: '[{"event":"event1"},{"event":"event2"}]' },
XML: {},
},
endpoint: '/endpoint',
files: {},
headers: { 'Content-Type': 'application/json' },
method: 'POST',
params: {},
type: 'REST',
version: '1',
};

const result = generateBatchedPayloadForArray(events);

expect(result).toEqual(expectedBatchedRequest);
});
});
});
30 changes: 18 additions & 12 deletions test/__tests__/data/mp_router_output.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,20 @@
"endpoint": "https://api.mixpanel.com/import/",
"headers": {
"Content-Type": "application/json",
"Content-Encoding": "gzip",
"Authorization": "Basic dGVzdF9hcGlfc2VjcmV0Og=="
},
"params": {
"strict": 1
},
"body": {
"JSON": {},
"JSON_ARRAY": {
"batch": "[{\"event\":\"Loaded a Page\",\"properties\":{\"ip\":\"0.0.0.0\",\"$user_id\":\"hjikl\",\"$current_url\":\"https://docs.rudderstack.com/destinations/mixpanel\",\"$screen_dpi\":2,\"mp_lib\":\"RudderLabs JavaScript SDK\",\"$app_build_number\":\"1.0.0\",\"$app_version_string\":\"1.0.5\",\"$insert_id\":\"dd266c67-9199-4a52-ba32-f46ddde67312\",\"token\":\"test_api_token\",\"distinct_id\":\"hjikl\",\"time\":1688624942,\"name\":\"Contact Us\",\"$browser\":\"Chrome\",\"$browser_version\":\"79.0.3945.117\"}}]"
},
"JSON_ARRAY": {},
"XML": {},
"FORM": {}
"FORM": {},
"GZIP": {
"payload": "H4sIAAAAAAAAE1VQQU7DMBD8SmRxbBKSpkmTazlBD4iKE0KWYy/t0sS2bKcgVf0765RKoL3NzM7O7NuZwQl0YB3bGqFAJSJ5FntgC2adseACgmfdmaElyX02D5F3kwfHURF4+MTjECE5OUdWfHJDhEOwvstzZaTP3KQUOB+EPGbSjLkCH1CLgEb7fMRvKzTMHl46AM2VRdaVCzZaPmBPbi+zwVb0PnkUJ7GTDm1Idg9PcUlYy/sJB8X1NPbgSF/cckbuRKfpEvfBod7/sqvIoqYa4dpDqbKuZd2kbdG2aSVWZdqLZZl+VLWi43WzLEraCeYImuSBKnBhkV+BBVMYO8nw/ysBR2BdUa/XdVm1FXXSIiJsYzS9IySvPgbpnfnyc/LNwZkR/mC3+MQ1LQVfttUqK4qGXS7vP4+w2Dm/AQAA"
}
},
"files": {}
},
Expand Down Expand Up @@ -76,18 +78,20 @@
"endpoint": "https://api.mixpanel.com/import/",
"headers": {
"Content-Type": "application/json",
"Content-Encoding": "gzip",
"Authorization": "Basic dGVzdF9hcGlfc2VjcmV0Og=="
},
"params": {
"strict": 1
},
"body": {
"JSON": {},
"JSON_ARRAY": {
"batch": "[{\"event\":\"Product Viewed\",\"properties\":{\"name\":\"T-Shirt\",\"revenue\":18.9,\"$user_id\":\"userId01\",\"$os\":\"iOS\",\"$screen_height\":1794,\"$screen_width\":1080,\"$screen_dpi\":420,\"$carrier\":\"Android\",\"$os_version\":\"8.1.0\",\"$device\":\"generic_x86\",\"$manufacturer\":\"Google\",\"$model\":\"Android SDK built for x86\",\"mp_device_model\":\"Android SDK built for x86\",\"$wifi\":true,\"$bluetooth_enabled\":false,\"mp_lib\":\"com.rudderstack.android.sdk.core\",\"$app_build_number\":\"1\",\"$app_version_string\":\"1.0\",\"$insert_id\":\"id2\",\"token\":\"test_api_token\",\"distinct_id\":\"userId01\",\"time\":1688624942,\"$device_id\":\"anonId01\"}}]"
},
"JSON_ARRAY": {},
"XML": {},
"FORM": {}
"FORM": {},
"GZIP": {
"payload": "H4sIAAAAAAAAE42RQUsDMRCF/8oSeqxLt5S67U0QRDwoVLyIhGwy7Q7dTZbJpBXE/+5kV+3Bi7fkmzcvkzevHwpO4Flt1RMFlywXLwhncGquBgoDECNEtf1Q3vQgquerXYvEUqbcmIRVdbmZq1mKQBqdaPLp3i0qEc2CNCt83OVztATgdQt4aOXF6nqzutAzOm4FLurFBboB1Xa1zMQaIgQStxvvKKCb3PUJKGLwwuuyKheZOjihzcMewAOh1e/1OvPe+LQ3lhONPnchHDoYC8FBd3EudrcPRZOw42IfqJi6+0FPvvo/6tkZ9zI6UwK5NF0CDoFbDd40naS73ZsuwujaYSNmNvQlJefkN2zssTSTeRndsbSBxjHNMOj8kNM+9c34h+qHf8egIxP6Q65MWaCXZfC0F3RLQRyOkONiiKzNgHoCc+UwMnrLf5bImFdfret6vVxtVsvfiCel8cGPys/Pty8LjzqIUAIAAA=="
}
},
"files": {}
}
Expand Down Expand Up @@ -145,18 +149,20 @@
"endpoint": "https://api.mixpanel.com/import/",
"headers": {
"Content-Type": "application/json",
"Content-Encoding": "gzip",
"Authorization": "Basic dGVzdF9hcGlfc2VjcmV0Og=="
},
"params": {
"strict": 1
},
"body": {
"JSON": {},
"JSON_ARRAY": {
"batch": "[{\"event\":\"$merge\",\"properties\":{\"$distinct_ids\":[\"test_user_id\",\"5094f5704b9cf2b3\"],\"token\":\"test_api_token\"}}]"
},
"JSON_ARRAY": {},
"XML": {},
"FORM": {}
"FORM": {},
"GZIP": {
"payload": "H4sIAAAAAAAAEyXMSQqAMBBE0bsULl2IA6JXEQkOpTRiDEnrRry7QbePz+9u8KJVtEh2+pVI4fzh6FUY0N5IZgkqdlIjc4QOyqDmDPQRYl1lTblUdVaOzbTkY4E+hR4bbVx+6eDE/PA8/QuxJ+OqcAAAAA=="
}
},
"files": {}
}
Expand Down

0 comments on commit fb1595f

Please sign in to comment.