Merge branch 'develop' into fix.tradeDesk

.github/workflows/build-push-docker-image.yml

@@ -23,6 +23,15 @@ on:
         type: string
       build_type:
         type: string
+      use_merge_sha:
+        type: boolean
+        default: false
+      skip_tests:
+        type: boolean
+        default: false
+        description: if this option is true, we would skip tests while building docker image
+      workflow_url:
+        type: string
     secrets:
       DOCKERHUB_PROD_TOKEN:
         required: true
@@ -31,18 +40,65 @@ env:
   DOCKERHUB_USERNAME: rudderlabs
 
 jobs:
+  get_sha:
+    runs-on: ubuntu-latest
+    name: Get SHA information
+    outputs:
+      sha: ${{steps.getSHA.outputs.SHA}}
+    steps:
+      - name: Checkout SHA
+        id: getSHA
+        run: |
+          if ${{inputs.use_merge_sha}} == true; then
+            sha=$(echo ${{github.sha}})
+          else
+            sha=$(echo ${{ github.event.pull_request.head.sha }})
+          fi
+          echo "SHA: $sha"
+          echo "SHA=$sha" >> $GITHUB_OUTPUT
+
+  get_changed_files:
+    runs-on: ubuntu-latest
+    name: Get Changed files
+    outputs:
+      should_execute_tests: ${{ steps.processing.outputs.should_execute_tests }}
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 1
+      - id: files
+        uses: Ana06/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          format: 'json'
+      - id: processing
+        run: |
+          readarray -t modified_files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.modified }}')"
+          echo "Modified files: $modified_files"
+          found=false
+          for modified_file in "${modified_files[@]}"; do
+            if [[ "$modified_file" == "Dockerfile" || "$modified_file" == "docker-compose.yml" || "$modified_file" == "Dockerfile" || "$modified_file" == "Dockerfile-ut-func" ]]; then
+              found=true
+              break
+            fi
+          done
+          echo "Match Found: $found"
+          echo "::set-output name=should_execute_tests::$found"
+
   build-transformer-image-arm64:
     name: Build Transformer Docker Image ARM64
     runs-on: [self-hosted, Linux, ARM64]
+    needs: [get_sha, get_changed_files]
     steps:
       - name: Checkout
         uses: actions/[email protected]
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
 
       - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3.4.0
+        uses: docker/setup-buildx-action@v3.6.1
 
       - name: Login to DockerHub
         uses: docker/[email protected]
@@ -62,6 +118,7 @@ jobs:
           # cache-to: type=gha,mode=max
 
       - name: Run Tests
+        if: ${{ inputs.skip_tests != true || needs.get_changed_files.outputs.should_execute_tests == true }}
         run: |
           docker run ${{ inputs.build_tag }} npm run test:js:ci
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
@@ -85,15 +142,16 @@ jobs:
   build-transformer-image-amd64:
     name: Build Transformer Docker Image AMD64
     runs-on: [self-hosted, Linux, X64]
+    needs: [get_sha, get_changed_files]
     steps:
       - name: Checkout
         uses: actions/[email protected]
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
 
       - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3.4.0
+        uses: docker/setup-buildx-action@v3.6.1
 
       - name: Login to DockerHub
         uses: docker/[email protected]
@@ -113,6 +171,7 @@ jobs:
           # cache-to: type=gha,mode=max
 
       - name: Run Tests
+        if: ${{ inputs.skip_tests != true || needs.get_changed_files.outputs.should_execute_tests == true }}
         run: |
           docker run ${{ inputs.build_tag }} npm run test:js:ci
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
@@ -140,7 +199,7 @@ jobs:
 
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.4.0
+        uses: docker/setup-buildx-action@v3.6.1
 
       - name: Login to DockerHub
         uses: docker/[email protected]

.github/workflows/dt-test-and-report-code-coverage.yml

@@ -13,10 +13,22 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  get_workflow_url:
+    runs-on: ubuntu-latest
+    steps:
+      - id: get_url
+        run: |
+          curl -s https://api.github.com/repos/${{ github.repository }}/actions/workflows/${{ github.workflow }}/runs/${{ github.run_id }} | jq -r .html_url >> workflow_url.txt
+          echo "::set-output name=workflow_url::$(cat workflow_url.txt)"
+    outputs:
+      url: ${{ steps.get_url.outputs.workflow_url }}
+
   coverage:
     name: Code Coverage
     runs-on: ubuntu-latest
-
+    needs: [get_workflow_url]
+    outputs:
+      tests_run_outcome: ${{steps.run_tests.outcome}}
     steps:
       - name: Checkout
         uses: actions/[email protected]
@@ -33,6 +45,8 @@ jobs:
         run: npm ci
 
       - name: Run Tests
+        id: run_tests
+        continue-on-error: true
         run: |
           # Supress logging in tests
           LOG_LEVEL=100 npm run test:js:ci
@@ -70,3 +84,13 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+  notify:
+    name: slack notification on failure
+    needs: [get_workflow_url, coverage]
+    if: needs.coverage.outputs.tests_run_outcome == 'failure' || failure()
+    uses: ./.github/workflows/slack-notify.yml
+    with:
+      workflow_url: ${{ needs.get_workflow_url.outputs.url }}
+      should_notify: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
+    secrets: inherit

.github/workflows/prepare-for-dev-deploy.yml

@@ -60,6 +60,7 @@ jobs:
       dockerfile: Dockerfile
       load_target: development
       push_target: production
+      use_merge_sha: true
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/prepare-for-prod-dt-deploy.yml

@@ -57,6 +57,8 @@ jobs:
       load_target: development
       push_target: production
       build_type: dt
+      use_merge_sha: true
+      skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/prepare-for-prod-ut-deploy.yml

@@ -60,6 +60,8 @@ jobs:
       load_target: development
       push_target: production
       build_type: ut
+      use_merge_sha: true
+      skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/prepare-for-staging-deploy.yml

@@ -52,6 +52,7 @@ jobs:
       dockerfile: Dockerfile
       load_target: development
       push_target: production
+      use_merge_sha: true
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/slack-notify.yml

@@ -0,0 +1,51 @@
+name: Notify workflow failure
+
+on:
+  workflow_call:
+    inputs:
+      should_notify:
+        type: boolean
+        default: true
+      workflow_url:
+        type: string
+        required: true
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.should_notify }}
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: notify
+        uses: slackapi/[email protected]
+        continue-on-error: true
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          PROJECT_NAME: 'Rudder Transformer'
+        with:
+          channel-id: ${{ secrets.SLACK_INTEGRATION_DEV_CHANNEL_ID }}
+          payload: |
+            {
+              "text": "*<prod release tests failed>*\nCC: <!subteam^S02AEQL26CT>",
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": ":siren2:  prod release tests - Failed  :siren2:"
+                  }
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "*<${{inputs.workflow_url}}|failed workflow>*\nCC: <!subteam^S02AEQL26CT>"
+                  }
+                }
+              ]
+            }

.github/workflows/verify.yml

@@ -28,13 +28,29 @@ jobs:
       - name: Install Dependencies
         run: npm ci
 
-      - name: Run Lint Checks
+      - id: files
+        uses: Ana06/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run format Checks
+        run: |
+          npx prettier ${{steps.files.outputs.added_modified}} --write
+
+      - run: git diff --exit-code
+
+      - name: Formatting Error message
+        if: ${{ failure() }}
+        run: |
+          echo 'prettier formatting failure. Ensure you run `npm run format` and commit the files.'
+
+      - name: Run eslint Checks
         run: |
-          npm run lint
+          npx eslint ${{steps.files.outputs.added_modified}} --fix
 
       - run: git diff --exit-code
 
-      - name: Error message
+      - name: Eslint Error message
         if: ${{ failure() }}
         run: |
-          echo 'Eslint check is failing Ensure you have run `npm run lint` and committed the files locally.'
+          echo 'Eslint failure. Ensure you run `npm run lint:fix` and commit the files.'

.vscode/settings.json

@@ -13,6 +13,10 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.formatOnSave": true
   },
+  "[json]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.formatOnSave": true
+  },
   "[yaml]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.formatOnSave": true

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.75.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.75.0...v1.75.1) (2024-08-14)
+
+
+### Bug Fixes
+
+* add validation for concurrent_modification error ([#3654](https://github.com/rudderlabs/rudder-transformer/issues/3654)) ([62cdc46](https://github.com/rudderlabs/rudder-transformer/commit/62cdc4641d44d79e21949722660173df4c749f24))
+* clevertap bugsnag issue ([#3656](https://github.com/rudderlabs/rudder-transformer/issues/3656)) ([6c51487](https://github.com/rudderlabs/rudder-transformer/commit/6c51487183b6b0b755620aac6cd51c2ffc966102))
+* snapchat conversion bugsnag issue ([#3657](https://github.com/rudderlabs/rudder-transformer/issues/3657)) ([31b03fc](https://github.com/rudderlabs/rudder-transformer/commit/31b03fc022ace0cda8df798c50e4764a9703c23b))
+* validation for iterable object of HS ([#3653](https://github.com/rudderlabs/rudder-transformer/issues/3653)) ([1cb3f86](https://github.com/rudderlabs/rudder-transformer/commit/1cb3f86c894f30bdf04df870484c6df3a956f36e))
+
+## [1.75.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.1...v1.75.0) (2024-08-12)
+
+
+### Features
+
+* move hubspot to transformer proxy to enable partial batch handling ([#3308](https://github.com/rudderlabs/rudder-transformer/issues/3308)) ([8450021](https://github.com/rudderlabs/rudder-transformer/commit/8450021672c51ac798ec0aeab422f5fceea5e53e))
+
+
+### Bug Fixes
+
+* handle attentive tag null, undefined properties ([#3647](https://github.com/rudderlabs/rudder-transformer/issues/3647)) ([9327925](https://github.com/rudderlabs/rudder-transformer/commit/932792561c98833baf9881a83ee36ae5000e37b4))
+* handle null values for braze dedupe ([#3638](https://github.com/rudderlabs/rudder-transformer/issues/3638)) ([0a9b681](https://github.com/rudderlabs/rudder-transformer/commit/0a9b68118241158623d45ee28896377296bafd91))
+
 ### [1.74.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.0...v1.74.1) (2024-08-08)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.74.1",
+  "version": "1.75.1",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/routerUtils.js

@@ -22,6 +22,7 @@ const userTransformHandler = () => {
 async function sendToDestination(destination, payload) {
   let parsedResponse;
   logger.info('Request recieved for destination', destination);
+
   const resp = await proxyRequest(payload);
 
   if (resp.success) {

src/services/comparator.ts

@@ -33,6 +33,7 @@ export class ComparatorService implements DestinationService {
 
   public init(): void {
     this.primaryService.init();
+
     this.secondaryService.init();
   }
 

src/v0/destinations/attentive_tag/transform.js

@@ -16,7 +16,7 @@ const {
 const {
   getDestinationItemProperties,
   getExternalIdentifiersMapping,
-  getPropertiesKeyValidation,
+  arePropertiesValid,
   validateTimestamp,
 } = require('./util');
 const { JSON_MIME_TYPE } = require('../../util/constant');
@@ -137,9 +137,9 @@ const trackResponseBuilder = (message, { Config }) => {
       payload = constructPayload(message, mappingConfig[ConfigCategory.TRACK.name]);
       endpoint = ConfigCategory.TRACK.endpoint;
       payload.type = get(message, 'event');
-      if (!getPropertiesKeyValidation(payload)) {
+      if (!arePropertiesValid(payload.properties)) {
         throw new InstrumentationError(
-          '[Attentive Tag]:The event name contains characters which is not allowed',
+          '[Attentive Tag]:The properties contains characters which is not allowed',
         );
       }
   }