chore(release): pull release/v1.86.0 into main

.github/workflows/build-push-docker-image.yml

@@ -1,4 +1,4 @@
 name: Build Transformer Docker Image
 
 on:
   workflow_call:
@@ -96,15 +96,15 @@
         with:
           ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
-
-      - name: Setup Docker Buildx
-        uses: docker/[email protected]
-
+
       - name: Login to DockerHub
         uses: docker/[email protected]
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}    
+
+      - name: Setup Docker Buildx
+        uses: docker/[email protected]
 
       - name: Build Docker Image
         uses: docker/[email protected]
@@ -149,16 +149,16 @@
         with:
           ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
-
-      - name: Setup Docker Buildx
-        uses: docker/[email protected]
-
+
       - name: Login to DockerHub
         uses: docker/[email protected]
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
+      - name: Setup Docker Buildx
+        uses: docker/[email protected]
+
       - name: Build Docker Image
         uses: docker/[email protected]
         with:
@@ -198,15 +198,15 @@
     needs: [build-transformer-image-amd64, build-transformer-image-arm64]
 
     steps:
-      - name: Set up Docker Buildx
-        uses: docker/[email protected]
-
       - name: Login to DockerHub
         uses: docker/[email protected]
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+
       - name: Create multi-arch manifest
         run: |
           docker buildx imagetools create -t ${{ inputs.push_tags }} ${{ inputs.push_tags }}-amd64 ${{ inputs.push_tags }}-arm64

.github/workflows/dt-test-and-report-code-coverage.yml

@@ -1,4 +1,4 @@
 name: DT Tests
 on:
   workflow_call:
     secrets:
@@ -33,7 +33,7 @@
       - name: Checkout
         uses: actions/[email protected]
         with:
-          fetch-depth: 1
+          fetch-depth: 0
 
       - name: Setup Node
         uses: actions/[email protected]

.github/workflows/prepare-for-staging-deploy.yml

@@ -1,4 +1,4 @@
 name: Prepare for Staging Environment Deployment
 
 on:
   pull_request:
@@ -78,6 +78,7 @@
     runs-on: ubuntu-latest
     needs: [generate-tag-names, build-transformer-image, build-user-transformer-image]
     env:
+      TF_IMAGE_REPOSITORY: rudderstack/rudder-transformer
       TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name }}
       UT_TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
     steps:
@@ -113,6 +114,7 @@
             git checkout -b $BRANCH_NAME
 
             cd helm-charts/shared-services/per-az/environment/staging
+            yq eval -i ".rudder-transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" staging.yaml
             yq eval -i ".rudder-transformer.image.tag=\"$TAG_NAME\"" staging.yaml
             yq eval -i ".user-transformer.image.tag=\"$TAG_NAME\"" staging.yaml
             git add staging.yaml

.github/workflows/verify.yml

@@ -1,4 +1,4 @@
 name: Verify
 
 on:
   pull_request:
@@ -35,8 +35,8 @@
 
       - name: Filter JS/TS Files
         run: |
-          echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' > changed_files.txt
-          if [ ! -s changed_files.txt ]; then
+          changed_files=$(echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' || true)
+          if [ -z "$changed_files" ]; then
             echo "No JS/TS files to format or lint."
             exit 0
           fi

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.86.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.1...v1.86.0) (2024-12-09)
+
+
+### Features
+
+* add support for getting anonymousId by note attributes array ([53d2bef](https://github.com/rudderlabs/rudder-transformer/commit/53d2bef38ad5d7bc5504111ec797b3c3973546dd))
+* add support for getting anonymousId by note attributes array ([#3893](https://github.com/rudderlabs/rudder-transformer/issues/3893)) ([d7f390c](https://github.com/rudderlabs/rudder-transformer/commit/d7f390cb471e44afb276484b8b804d1f257c539c))
+
+
+### Bug Fixes
+
+* braze subscription group fixes ([#3901](https://github.com/rudderlabs/rudder-transformer/issues/3901)) ([ebcf84e](https://github.com/rudderlabs/rudder-transformer/commit/ebcf84e07bf121d882c99df973af265a915a1ce1))
+* remove redundant ids and userIdentifier when gbraid or wbraid are there ([#3910](https://github.com/rudderlabs/rudder-transformer/issues/3910)) ([313710c](https://github.com/rudderlabs/rudder-transformer/commit/313710ca725538e5ffe357216d9c88e444f995c8))
+* skipping users events for snowpipe streaming ([#3836](https://github.com/rudderlabs/rudder-transformer/issues/3836)) ([12621c8](https://github.com/rudderlabs/rudder-transformer/commit/12621c8eee641f5a03a997e95ed016cff0eefde7))
+
 ### [1.85.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.0...v1.85.1) (2024-11-21)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.85.1",
+  "version": "1.86.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/util/prometheus.js

@@ -520,6 +520,24 @@ class Prometheus {
         type: 'counter',
         labelNames: ['destination_id'],
       },
+      {
+        name: 'braze_batch_subscription_size',
+        help: 'braze_batch_subscription_size',
+        type: 'gauge',
+        labelNames: ['destination_id'],
+      },
+      {
+        name: 'braze_batch_subscription_combined_size',
+        help: 'braze_batch_subscription_combined_size',
+        type: 'gauge',
+        labelNames: ['destination_id'],
+      },
+      {
+        name: 'mailjet_packing_size',
+        help: 'mailjet_packing_size',
+        type: 'gauge',
+        labelNames: ['group'],
+      },
       {
         name: 'hs_batch_size',
         help: 'hs_batch_size',

src/v0/destinations/braze/braze.util.test.js

@@ -6,6 +6,7 @@ const {
   getPurchaseObjs,
   setAliasObject,
   handleReservedProperties,
+  combineSubscriptionGroups,
 } = require('./util');
 const { processBatch } = require('./util');
 const {
@@ -958,7 +959,9 @@ describe('processBatch', () => {
               attributes: [{ id: i, name: 'test', xyz: 'abc' }],
               events: [{ id: i, event: 'test', xyz: 'abc' }],
               purchases: [{ id: i, purchase: 'test', xyz: 'abc' }],
-              subscription_groups: [{ id: i, group: 'test', xyz: 'abc' }],
+              subscription_groups: [
+                { subscription_group_id: i, group: 'test', subscription_state: 'abc' },
+              ],
               merge_updates: [{ id: i, alias: 'test', xyz: 'abc' }],
             },
           },
@@ -972,7 +975,7 @@ describe('processBatch', () => {
 
     // Assert that the response is as expected
     expect(result.length).toBe(1); // One successful batched request and one failure response
-    expect(result[0].batchedRequest.length).toBe(6); // Two batched requests
+    expect(result[0].batchedRequest.length).toBe(8); // Two batched requests
     expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name
     expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes
     expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events
@@ -981,10 +984,12 @@ describe('processBatch', () => {
     expect(result[0].batchedRequest[1].body.JSON.attributes.length).toBe(25); // Second batch contains remaining 25 attributes
     expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(25); // Second batch contains remaining 25 events
     expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(25); // Second batch contains remaining 25 purchases
-    expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group
-    expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50);
-    expect(result[0].batchedRequest[4].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates
-    expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates
+    expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group
+    expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group
+    expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Third batch contains 25 subscription group
+    expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(25); // Fourth batch contains 25 subscription group
+    expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates
+    expect(result[0].batchedRequest[7].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates
   });
 
   test('processBatch handles more than 75 attributes, events, and purchases with non uniform distribution', () => {
@@ -1055,7 +1060,9 @@ describe('processBatch', () => {
       batchedRequest: {
         body: {
           JSON: {
-            subscription_groups: [{ id: i, group: 'test', xyz: 'abc' }],
+            subscription_groups: [
+              { subscription_group_id: i, group: 'test', subscription_state: 'abc' },
+            ],
           },
         },
       },
@@ -1093,7 +1100,7 @@ describe('processBatch', () => {
     // Assert that the response is as expected
     expect(result.length).toBe(1); // One successful batched request and one failure response
     expect(result[0].metadata.length).toBe(490); // Check the total length is same as input jobs (120 + 160 + 100 + 70 +40)
-    expect(result[0].batchedRequest.length).toBe(6); // Two batched requests
+    expect(result[0].batchedRequest.length).toBe(7); // Two batched requests
     expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name
     expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes
     expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events
@@ -1103,9 +1110,10 @@ describe('processBatch', () => {
     expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(45); // Second batch contains remaining 45 events
     expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(75); // Second batch contains remaining 75 purchases
     expect(result[0].batchedRequest[2].body.JSON.purchases.length).toBe(10); // Third batch contains remaining 10 purchases
-    expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group
-    expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(20); // Second batch contains 20 subscription group
-    expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(40); // First batch contains 40 merge_updates
+    expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group
+    expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group
+    expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(20); // Third batch contains 20 subscription group
+    expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(40); // First batch contains 50 merge_updates
   });
 
   test('check success and failure scenarios both for processBatch', () => {
@@ -1751,3 +1759,124 @@ describe('handleReservedProperties', () => {
     });
   });
 });
+
+describe('combineSubscriptionGroups', () => {
+  it('should merge external_ids, emails, and phones for the same subscription_group_id and subscription_state', () => {
+    const input = [
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        external_ids: ['id1', 'id2'],
+        emails: ['[email protected]', '[email protected]'],
+        phones: ['+1234567890', '+0987654321'],
+      },
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        external_ids: ['id2', 'id3'],
+        emails: ['[email protected]', '[email protected]'],
+        phones: ['+1234567890', '+1122334455'],
+      },
+    ];
+
+    const expectedOutput = [
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        external_ids: ['id1', 'id2', 'id3'],
+        emails: ['[email protected]', '[email protected]', '[email protected]'],
+        phones: ['+1234567890', '+0987654321', '+1122334455'],
+      },
+    ];
+
+    const result = combineSubscriptionGroups(input);
+    expect(result).toEqual(expectedOutput);
+  });
+
+  it('should handle groups with missing external_ids, emails, or phones', () => {
+    const input = [
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        external_ids: ['id1'],
+      },
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        emails: ['[email protected]'],
+      },
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        phones: ['+1234567890'],
+      },
+    ];
+
+    const expectedOutput = [
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        external_ids: ['id1'],
+        emails: ['[email protected]'],
+        phones: ['+1234567890'],
+      },
+    ];
+
+    const result = combineSubscriptionGroups(input);
+    expect(result).toEqual(expectedOutput);
+  });
+
+  it('should handle multiple unique subscription groups', () => {
+    const input = [
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        external_ids: ['id1'],
+      },
+      {
+        subscription_group_id: 'group2',
+        subscription_state: 'Unsubscribed',
+        external_ids: ['id2'],
+        emails: ['[email protected]'],
+      },
+    ];
+
+    const expectedOutput = [
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        external_ids: ['id1'],
+      },
+      {
+        subscription_group_id: 'group2',
+        subscription_state: 'Unsubscribed',
+        external_ids: ['id2'],
+        emails: ['[email protected]'],
+      },
+    ];
+
+    const result = combineSubscriptionGroups(input);
+    expect(result).toEqual(expectedOutput);
+  });
+
+  it('should not include undefined fields in the output', () => {
+    const input = [
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        external_ids: ['id1'],
+      },
+    ];
+
+    const expectedOutput = [
+      {
+        subscription_group_id: 'group1',
+        subscription_state: 'Subscribed',
+        external_ids: ['id1'],
+      },
+    ];
+
+    const result = combineSubscriptionGroups(input);
+    expect(result).toEqual(expectedOutput);
+  });
+});

src/v0/destinations/braze/config.js

@@ -36,7 +36,7 @@ const IDENTIFY_BRAZE_MAX_REQ_COUNT = 50;
 // https://www.braze.com/docs/api/endpoints/user_data/post_user_delete/
 
 const ALIAS_BRAZE_MAX_REQ_COUNT = 50;
-const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 50;
+const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 25;
 
 const DEL_MAX_BATCH_SIZE = 50;
 const DESTINATION = 'braze';

src/v0/destinations/braze/transform.js

@@ -383,7 +383,7 @@ function processGroup(message, destination) {
       );
     }
     subscriptionGroup.subscription_state = message.traits.subscriptionState;
-    subscriptionGroup.external_id = [message.userId];
+    subscriptionGroup.external_ids = [message.userId];
     const phone = getFieldValueFromMessage(message, 'phone');
     const email = getFieldValueFromMessage(message, 'email');
     if (phone) {