chore(release): pull main into develop post release v1.46.1 (#2751)

rudderlabs · Oct 19, 2023 · 81066ed · 81066ed
2 parents a2ccdad + 4d65048
commit 81066ed
Show file tree

Hide file tree

Showing 14 changed files with 1,011 additions and 28 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,19 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.46.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.46.1...v1.46.2) (2023-10-19)
+
+
+### Bug Fixes
+
+* do not send 298 event for aborted ones in ut ([d414064](https://github.com/rudderlabs/rudder-transformer/commit/d414064aac1c8d769607346bc48b8db549e7a8dc))
+
+### [1.46.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.46.0...v1.46.1) (2023-10-19)
+
+### Bug Fixes
+
+* add gzip support compatibility ([#2745](https://github.com/rudderlabs/rudder-transformer/issues/2745)) ([d72410f](https://github.com/rudderlabs/rudder-transformer/commit/d72410f7962e702218be3c2414de62341ca29e39))
+
 ## [1.46.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.45.3...v1.46.0) (2023-10-18)
 
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.46.0",
+  "version": "1.46.2",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

diff --git a/src/services/userTransform.ts b/src/services/userTransform.ts
@@ -96,6 +96,12 @@ export default class UserTransformService {
 
           const transformedEventsWithMetadata: ProcessorTransformationResponse[] = [];
           destTransformedEvents.forEach((ev) => {
+            // add messageId to output set
+            if (ev.metadata?.messageId) {
+              messageIdsInOutputSet.add(ev.metadata.messageId);
+            } else if (ev.metadata?.messageIds) {
+              ev.metadata.messageIds.forEach((id) => messageIdsInOutputSet.add(id));
+            }
             if (ev.error) {
               transformedEventsWithMetadata.push({
                 statusCode: 400,
@@ -114,12 +120,6 @@ export default class UserTransformService {
               } as ProcessorTransformationResponse);
               return;
             }
-            // add messageId to output set
-            if (ev.metadata?.messageId) {
-              messageIdsInOutputSet.add(ev.metadata.messageId);
-            } else if (ev.metadata?.messageIds) {
-              ev.metadata.messageIds.forEach((id) => messageIdsInOutputSet.add(id));
-            }
             transformedEventsWithMetadata.push({
               output: ev.transformedEvent,
               metadata: isEmpty(ev.metadata) ? commonMetadata : ev.metadata,

diff --git a/src/v0/destinations/braze/transform.js b/src/v0/destinations/braze/transform.js
@@ -218,7 +218,7 @@ async function processIdentify(message, destination) {
   );
   if (!isHttpStatusSuccess(brazeIdentifyResp.status)) {
     throw new NetworkError(
-      'Braze identify failed',
+      `Braze identify failed - ${JSON.stringify(brazeIdentifyResp.response)}`,
       brazeIdentifyResp.status,
       {
         [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(brazeIdentifyResp.status),

diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js
@@ -455,10 +455,10 @@ const processRouterDest = async (inputs, reqMetadata) => {
       const { engageEvents, groupsEvents, trackEvents, importEvents, batchErrorRespList } =
         groupEventsByEndpoint(transformedPayloads);
 
-      const engageRespList = batchEvents(engageEvents, ENGAGE_MAX_BATCH_SIZE);
-      const groupsRespList = batchEvents(groupsEvents, GROUPS_MAX_BATCH_SIZE);
-      const trackRespList = batchEvents(trackEvents, TRACK_MAX_BATCH_SIZE);
-      const importRespList = batchEvents(importEvents, IMPORT_MAX_BATCH_SIZE);
+      const engageRespList = batchEvents(engageEvents, ENGAGE_MAX_BATCH_SIZE, reqMetadata);
+      const groupsRespList = batchEvents(groupsEvents, GROUPS_MAX_BATCH_SIZE, reqMetadata);
+      const trackRespList = batchEvents(trackEvents, TRACK_MAX_BATCH_SIZE, reqMetadata);
+      const importRespList = batchEvents(importEvents, IMPORT_MAX_BATCH_SIZE, reqMetadata);
       const batchSuccessRespList = [
         ...engageRespList,
         ...groupsRespList,

diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js
@@ -11,6 +11,7 @@ const {
   batchMultiplexedEvents,
   getSuccessRespEvents,
   defaultBatchRequestConfig,
+  IsGzipSupported,
 } = require('../../util');
 const {
   ConfigCategory,
@@ -212,16 +213,17 @@ const groupEventsByEndpoint = (events) => {
   };
 };
 
-const generateBatchedPayloadForArray = (events) => {
+const generateBatchedPayloadForArray = (events, reqMetadata) => {
   const { batchedRequest } = defaultBatchRequestConfig();
   const firstEvent = events[0];
   batchedRequest.endpoint = firstEvent.endpoint;
   batchedRequest.headers = firstEvent.headers;
   batchedRequest.params = firstEvent.params;
 
   const batchResponseList = events.flatMap((event) => JSON.parse(event.body.JSON_ARRAY.batch));
-  // Gzipping the payload for /import endpoint
-  if (firstEvent.endpoint.includes('import')) {
+
+  if (IsGzipSupported(reqMetadata) && firstEvent.endpoint.includes('import')) {
+    // Gzipping the payload for /import endpoint
     batchedRequest.body.GZIP = { payload: JSON.stringify(batchResponseList) };
   } else {
     batchedRequest.body.JSON_ARRAY = { batch: JSON.stringify(batchResponseList) };
@@ -230,10 +232,10 @@ const generateBatchedPayloadForArray = (events) => {
   return batchedRequest;
 };
 
-const batchEvents = (successRespList, maxBatchSize) => {
+const batchEvents = (successRespList, maxBatchSize, reqMetadata) => {
   const batchedEvents = batchMultiplexedEvents(successRespList, maxBatchSize);
   return batchedEvents.map((batch) => {
-    const batchedRequest = generateBatchedPayloadForArray(batch.events);
+    const batchedRequest = generateBatchedPayloadForArray(batch.events, reqMetadata);
     return getSuccessRespEvents(batchedRequest, batch.metadata, batch.destination, true);
   });
 };

diff --git a/src/v0/destinations/mp/util.test.js b/src/v0/destinations/mp/util.test.js
@@ -4,6 +4,7 @@ const {
   batchEvents,
   generateBatchedPayloadForArray,
 } = require('./util');
+const { FEATURE_GZIP_SUPPORT } = require('../../util/constant');
 
 const destinationMock = {
   Config: {
@@ -524,7 +525,9 @@ describe('Mixpanel utils test', () => {
         version: '1',
       };
 
-      const result = generateBatchedPayloadForArray(events);
+      const result = generateBatchedPayloadForArray(events, {
+        features: { [FEATURE_GZIP_SUPPORT]: true },
+      });
 
       expect(result).toEqual(expectedBatchedRequest);
     });
@@ -560,7 +563,9 @@ describe('Mixpanel utils test', () => {
         version: '1',
       };
 
-      const result = generateBatchedPayloadForArray(events);
+      const result = generateBatchedPayloadForArray(events, {
+        features: { [FEATURE_GZIP_SUPPORT]: true },
+      });
 
       expect(result).toEqual(expectedBatchedRequest);
     });

diff --git a/src/v0/util/constant.js b/src/v0/util/constant.js
@@ -11,6 +11,7 @@ const API_CALL = 'api_call_count';
 const JSON_MIME_TYPE = 'application/json';
 
 const FEATURE_FILTER_CODE = 'filter-code';
+const FEATURE_GZIP_SUPPORT = 'gzip-support';
 
 const HTTP_STATUS_CODES = {
   // 1xx Informational
@@ -89,4 +90,5 @@ module.exports = {
   HTTP_STATUS_CODES,
   USER_LEAD_CACHE_TTL,
   FEATURE_FILTER_CODE,
+  FEATURE_GZIP_SUPPORT,
 };
diff --git a/src/v0/util/index.js b/src/v0/util/index.js
@@ -32,7 +32,8 @@ const {
   REFRESH_TOKEN,
   AUTH_STATUS_INACTIVE,
 } = require('../../adapters/networkhandler/authConstants');
-const { FEATURE_FILTER_CODE } = require('./constant');
+const { FEATURE_FILTER_CODE, FEATURE_GZIP_SUPPORT } = require('./constant');
+
 // ========================================================================
 // INLINERS
 // ========================================================================
@@ -372,7 +373,7 @@ const hashToSha256 = (value) => sha256(value);
 
 // Check what type of gender and convert to f or m
 const getFbGenderVal = (gender) => {
-  if (typeof (gender) !== 'string') {
+  if (typeof gender !== 'string') {
     return null;
   }
   if (
@@ -2050,11 +2051,20 @@ const getAuthErrCategoryFromStCode = (status) => {
   return '';
 };
 
-const validateEventType = event => {
-  if(!event || typeof event !== "string"){
-    throw new InstrumentationError("Event is a required field and should be a string");
+const validateEventType = (event) => {
+  if (!event || typeof event !== 'string') {
+    throw new InstrumentationError('Event is a required field and should be a string');
   }
-}
+};
+
+const IsGzipSupported = (reqMetadata = {}) => {
+  if (reqMetadata && typeof reqMetadata === 'object' && !Array.isArray(reqMetadata)) {
+    const { features } = reqMetadata;
+    return !!features?.[FEATURE_GZIP_SUPPORT];
+  }
+  return false;
+};
+
 // ========================================================================
 // EXPORTS
 // ========================================================================
@@ -2161,4 +2171,5 @@ module.exports = {
   getAuthErrCategoryFromErrDetailsAndStCode,
   getAuthErrCategoryFromStCode,
   isNewStatusCodesAccepted,
+  IsGzipSupported,
 };
diff --git a/test/__tests__/data/user_transformation_service_filter_input.json b/test/__tests__/data/user_transformation_service_filter_input.json
@@ -181,5 +181,67 @@
         }
       ]
     }
+  },
+  {
+    "message": {
+      "channel": "web",
+      "context": {
+        "app": {
+          "build": "1.0.0",
+          "name": "RudderLabs JavaScript SDK",
+          "namespace": "com.rudderlabs.javascript",
+          "version": "1.0.0"
+        },
+        "traits": {
+          "email": "[email protected]",
+          "anonymousId": "12345"
+        },
+        "library": {
+          "name": "RudderLabs JavaScript SDK",
+          "version": "1.0.0"
+        },
+        "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",
+        "locale": "en-US",
+        "ip": "0.0.0.0",
+        "os": {
+          "name": "",
+          "version": ""
+        },
+        "screen": {
+          "density": 2
+        }
+      },
+      "type": "non-standard",
+      "messageId": "ec5481b6-a926-4d2e-b293-0b3a77c4d3be",
+      "originalTimestamp": "2019-10-14T11:15:18.300Z",
+      "anonymousId": "00000000000000000000000000",
+      "userId": "12345",
+      "event": "test track event GA3",
+      "properties": {
+        "user_actual_role": "system_admin, system_user",
+        "user_actual_id": 12345
+      },
+      "integrations": {
+        "All": true
+      },
+      "sentAt": "2019-10-14T11:15:53.296Z"
+    },
+    "metadata": {
+      "messageId": "ec5481b6-a926-4d2e-b293-0b3a77c4d3be",
+      "sourceId": "s1",
+      "destinationId": "d1"
+    },
+    "destination": {
+      "ID": 2,
+      "Config": {
+        "trackingID": "abcd"
+      },
+      "Enabled": true,
+      "Transformations": [
+        {
+          "VersionID": "24"
+        }
+      ]
+    }
   }
 ]
diff --git a/test/__tests__/data/user_transformation_service_filter_output.json b/test/__tests__/data/user_transformation_service_filter_output.json
@@ -68,6 +68,15 @@
         "destinationId": "d1"
       },
       "statusCode": 200
+    },
+    {
+      "error": "Error: non-standard event\n    at transformEvent (base transformation:5:56)",
+      "metadata": {
+        "destinationId": "d1",
+        "messageId": "ec5481b6-a926-4d2e-b293-0b3a77c4d3be",
+        "sourceId": "s1"
+      },
+      "statusCode": 400
     }
   ],
   "retryStatus": 200

diff --git a/test/__tests__/user_transformation_ts.test.ts b/test/__tests__/user_transformation_ts.test.ts
@@ -8,7 +8,7 @@ const name = 'User Transformations';
 const randomID = () => Math.random().toString(36).substring(2, 15);
 
 describe('User Transform Service', () => {
-  fit(`Filtering ${name} Test`, async () => {
+  it(`Filtering ${name} Test`, async () => {
     const versionId = '24'; // set in input file
     const inputData = require(`./data/${integration}_filter_input.json`);
     const expectedData = require(`./data/${integration}_filter_output.json`);
@@ -21,6 +21,7 @@ describe('User Transform Service', () => {
                 const eventType = event.type;
                 log(eventType);
                 log(eventType.match(/track/g));
+                if(eventType === 'non-standard') throw new Error('non-standard event');
                 if(eventType && !eventType.match(/track/g)) return;
                 return event;
               }