Skip to content

Commit

Permalink
chore(release): pull release/v1.86.0 into main (#3925)
Browse files Browse the repository at this point in the history
  • Loading branch information
utsabc authored Dec 10, 2024
2 parents 275e971 + 6a0f475 commit 0965f30
Showing 60 changed files with 2,927 additions and 288 deletions.
24 changes: 12 additions & 12 deletions .github/workflows/build-push-docker-image.yml
Original file line number Diff line number Diff line change
@@ -96,15 +96,15 @@ jobs:
with:
ref: ${{ needs.get_sha.outputs.sha }}
fetch-depth: 1

- name: Setup Docker Buildx
uses: docker/[email protected]


- name: Login to DockerHub
uses: docker/[email protected]
with:
username: ${{ env.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}

- name: Setup Docker Buildx
uses: docker/[email protected]

- name: Build Docker Image
uses: docker/[email protected]
@@ -149,16 +149,16 @@ jobs:
with:
ref: ${{ needs.get_sha.outputs.sha }}
fetch-depth: 1

- name: Setup Docker Buildx
uses: docker/[email protected]


- name: Login to DockerHub
uses: docker/[email protected]
with:
username: ${{ env.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}

- name: Setup Docker Buildx
uses: docker/[email protected]

- name: Build Docker Image
uses: docker/[email protected]
with:
@@ -198,15 +198,15 @@ jobs:
needs: [build-transformer-image-amd64, build-transformer-image-arm64]

steps:
- name: Set up Docker Buildx
uses: docker/[email protected]

- name: Login to DockerHub
uses: docker/[email protected]
with:
username: ${{ env.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}

- name: Set up Docker Buildx
uses: docker/[email protected]

- name: Create multi-arch manifest
run: |
docker buildx imagetools create -t ${{ inputs.push_tags }} ${{ inputs.push_tags }}-amd64 ${{ inputs.push_tags }}-arm64
2 changes: 1 addition & 1 deletion .github/workflows/dt-test-and-report-code-coverage.yml
Original file line number Diff line number Diff line change
@@ -33,7 +33,7 @@ jobs:
- name: Checkout
uses: actions/[email protected]
with:
fetch-depth: 1
fetch-depth: 0

- name: Setup Node
uses: actions/[email protected]
2 changes: 2 additions & 0 deletions .github/workflows/prepare-for-staging-deploy.yml
Original file line number Diff line number Diff line change
@@ -78,6 +78,7 @@ jobs:
runs-on: ubuntu-latest
needs: [generate-tag-names, build-transformer-image, build-user-transformer-image]
env:
TF_IMAGE_REPOSITORY: rudderstack/rudder-transformer
TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name }}
UT_TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
steps:
@@ -113,6 +114,7 @@ jobs:
git checkout -b $BRANCH_NAME
cd helm-charts/shared-services/per-az/environment/staging
yq eval -i ".rudder-transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" staging.yaml
yq eval -i ".rudder-transformer.image.tag=\"$TAG_NAME\"" staging.yaml
yq eval -i ".user-transformer.image.tag=\"$TAG_NAME\"" staging.yaml
git add staging.yaml
4 changes: 2 additions & 2 deletions .github/workflows/verify.yml
Original file line number Diff line number Diff line change
@@ -35,8 +35,8 @@ jobs:

- name: Filter JS/TS Files
run: |
echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' > changed_files.txt
if [ ! -s changed_files.txt ]; then
changed_files=$(echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' || true)
if [ -z "$changed_files" ]; then
echo "No JS/TS files to format or lint."
exit 0
fi
15 changes: 15 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -2,6 +2,21 @@

All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.

## [1.86.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.1...v1.86.0) (2024-12-09)


### Features

* add support for getting anonymousId by note attributes array ([53d2bef](https://github.com/rudderlabs/rudder-transformer/commit/53d2bef38ad5d7bc5504111ec797b3c3973546dd))
* add support for getting anonymousId by note attributes array ([#3893](https://github.com/rudderlabs/rudder-transformer/issues/3893)) ([d7f390c](https://github.com/rudderlabs/rudder-transformer/commit/d7f390cb471e44afb276484b8b804d1f257c539c))


### Bug Fixes

* braze subscription group fixes ([#3901](https://github.com/rudderlabs/rudder-transformer/issues/3901)) ([ebcf84e](https://github.com/rudderlabs/rudder-transformer/commit/ebcf84e07bf121d882c99df973af265a915a1ce1))
* remove redundant ids and userIdentifier when gbraid or wbraid are there ([#3910](https://github.com/rudderlabs/rudder-transformer/issues/3910)) ([313710c](https://github.com/rudderlabs/rudder-transformer/commit/313710ca725538e5ffe357216d9c88e444f995c8))
* skipping users events for snowpipe streaming ([#3836](https://github.com/rudderlabs/rudder-transformer/issues/3836)) ([12621c8](https://github.com/rudderlabs/rudder-transformer/commit/12621c8eee641f5a03a997e95ed016cff0eefde7))

### [1.85.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.0...v1.85.1) (2024-11-21)


4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "rudder-transformer",
"version": "1.85.1",
"version": "1.86.0",
"description": "",
"homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
"bugs": {
18 changes: 18 additions & 0 deletions src/util/prometheus.js
Original file line number Diff line number Diff line change
@@ -520,6 +520,24 @@ class Prometheus {
type: 'counter',
labelNames: ['destination_id'],
},
{
name: 'braze_batch_subscription_size',
help: 'braze_batch_subscription_size',
type: 'gauge',
labelNames: ['destination_id'],
},
{
name: 'braze_batch_subscription_combined_size',
help: 'braze_batch_subscription_combined_size',
type: 'gauge',
labelNames: ['destination_id'],
},
{
name: 'mailjet_packing_size',
help: 'mailjet_packing_size',
type: 'gauge',
labelNames: ['group'],
},
{
name: 'hs_batch_size',
help: 'hs_batch_size',
151 changes: 140 additions & 11 deletions src/v0/destinations/braze/braze.util.test.js
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ const {
getPurchaseObjs,
setAliasObject,
handleReservedProperties,
combineSubscriptionGroups,
} = require('./util');
const { processBatch } = require('./util');
const {
@@ -958,7 +959,9 @@ describe('processBatch', () => {
attributes: [{ id: i, name: 'test', xyz: 'abc' }],
events: [{ id: i, event: 'test', xyz: 'abc' }],
purchases: [{ id: i, purchase: 'test', xyz: 'abc' }],
subscription_groups: [{ id: i, group: 'test', xyz: 'abc' }],
subscription_groups: [
{ subscription_group_id: i, group: 'test', subscription_state: 'abc' },
],
merge_updates: [{ id: i, alias: 'test', xyz: 'abc' }],
},
},
@@ -972,7 +975,7 @@ describe('processBatch', () => {

// Assert that the response is as expected
expect(result.length).toBe(1); // One successful batched request and one failure response
expect(result[0].batchedRequest.length).toBe(6); // Two batched requests
expect(result[0].batchedRequest.length).toBe(8); // Two batched requests
expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name
expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes
expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events
@@ -981,10 +984,12 @@ describe('processBatch', () => {
expect(result[0].batchedRequest[1].body.JSON.attributes.length).toBe(25); // Second batch contains remaining 25 attributes
expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(25); // Second batch contains remaining 25 events
expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(25); // Second batch contains remaining 25 purchases
expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group
expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50);
expect(result[0].batchedRequest[4].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates
expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates
expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group
expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group
expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Third batch contains 25 subscription group
expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(25); // Fourth batch contains 25 subscription group
expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates
expect(result[0].batchedRequest[7].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates
});

test('processBatch handles more than 75 attributes, events, and purchases with non uniform distribution', () => {
@@ -1055,7 +1060,9 @@ describe('processBatch', () => {
batchedRequest: {
body: {
JSON: {
subscription_groups: [{ id: i, group: 'test', xyz: 'abc' }],
subscription_groups: [
{ subscription_group_id: i, group: 'test', subscription_state: 'abc' },
],
},
},
},
@@ -1093,7 +1100,7 @@ describe('processBatch', () => {
// Assert that the response is as expected
expect(result.length).toBe(1); // One successful batched request and one failure response
expect(result[0].metadata.length).toBe(490); // Check the total length is same as input jobs (120 + 160 + 100 + 70 +40)
expect(result[0].batchedRequest.length).toBe(6); // Two batched requests
expect(result[0].batchedRequest.length).toBe(7); // Two batched requests
expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name
expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes
expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events
@@ -1103,9 +1110,10 @@ describe('processBatch', () => {
expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(45); // Second batch contains remaining 45 events
expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(75); // Second batch contains remaining 75 purchases
expect(result[0].batchedRequest[2].body.JSON.purchases.length).toBe(10); // Third batch contains remaining 10 purchases
expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group
expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(20); // Second batch contains 20 subscription group
expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(40); // First batch contains 40 merge_updates
expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group
expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group
expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(20); // Third batch contains 20 subscription group
expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(40); // First batch contains 50 merge_updates
});

test('check success and failure scenarios both for processBatch', () => {
@@ -1751,3 +1759,124 @@ describe('handleReservedProperties', () => {
});
});
});

describe('combineSubscriptionGroups', () => {
it('should merge external_ids, emails, and phones for the same subscription_group_id and subscription_state', () => {
const input = [
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
external_ids: ['id1', 'id2'],
emails: ['[email protected]', '[email protected]'],
phones: ['+1234567890', '+0987654321'],
},
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
external_ids: ['id2', 'id3'],
emails: ['[email protected]', '[email protected]'],
phones: ['+1234567890', '+1122334455'],
},
];

const expectedOutput = [
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
external_ids: ['id1', 'id2', 'id3'],
emails: ['[email protected]', '[email protected]', '[email protected]'],
phones: ['+1234567890', '+0987654321', '+1122334455'],
},
];

const result = combineSubscriptionGroups(input);
expect(result).toEqual(expectedOutput);
});

it('should handle groups with missing external_ids, emails, or phones', () => {
const input = [
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
external_ids: ['id1'],
},
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
emails: ['[email protected]'],
},
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
phones: ['+1234567890'],
},
];

const expectedOutput = [
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
external_ids: ['id1'],
emails: ['[email protected]'],
phones: ['+1234567890'],
},
];

const result = combineSubscriptionGroups(input);
expect(result).toEqual(expectedOutput);
});

it('should handle multiple unique subscription groups', () => {
const input = [
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
external_ids: ['id1'],
},
{
subscription_group_id: 'group2',
subscription_state: 'Unsubscribed',
external_ids: ['id2'],
emails: ['[email protected]'],
},
];

const expectedOutput = [
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
external_ids: ['id1'],
},
{
subscription_group_id: 'group2',
subscription_state: 'Unsubscribed',
external_ids: ['id2'],
emails: ['[email protected]'],
},
];

const result = combineSubscriptionGroups(input);
expect(result).toEqual(expectedOutput);
});

it('should not include undefined fields in the output', () => {
const input = [
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
external_ids: ['id1'],
},
];

const expectedOutput = [
{
subscription_group_id: 'group1',
subscription_state: 'Subscribed',
external_ids: ['id1'],
},
];

const result = combineSubscriptionGroups(input);
expect(result).toEqual(expectedOutput);
});
});
2 changes: 1 addition & 1 deletion src/v0/destinations/braze/config.js
Original file line number Diff line number Diff line change
@@ -36,7 +36,7 @@ const IDENTIFY_BRAZE_MAX_REQ_COUNT = 50;
// https://www.braze.com/docs/api/endpoints/user_data/post_user_delete/

const ALIAS_BRAZE_MAX_REQ_COUNT = 50;
const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 50;
const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 25;

const DEL_MAX_BATCH_SIZE = 50;
const DESTINATION = 'braze';
2 changes: 1 addition & 1 deletion src/v0/destinations/braze/transform.js
Original file line number Diff line number Diff line change
@@ -383,7 +383,7 @@ function processGroup(message, destination) {
);
}
subscriptionGroup.subscription_state = message.traits.subscriptionState;
subscriptionGroup.external_id = [message.userId];
subscriptionGroup.external_ids = [message.userId];
const phone = getFieldValueFromMessage(message, 'phone');
const email = getFieldValueFromMessage(message, 'email');
if (phone) {
Loading

0 comments on commit 0965f30

Please sign in to comment.