Merge branch 'develop' of github.com:rudderlabs/rudder-transformer in…

…to chore.conflict-main-develop

.github/workflows/build-push-docker-image.yml

@@ -23,6 +23,15 @@ on:
         type: string
       build_type:
         type: string
+      use_merge_sha:
+        type: boolean
+        default: false
+      skip_tests:
+        type: boolean
+        default: false
+        description: if this option is true, we would skip tests while building docker image
+      workflow_url:
+        type: string
     secrets:
       DOCKERHUB_PROD_TOKEN:
         required: true
@@ -31,27 +40,74 @@ env:
   DOCKERHUB_USERNAME: rudderlabs
 
 jobs:
+  get_sha:
+    runs-on: ubuntu-latest
+    name: Get SHA information
+    outputs:
+      sha: ${{steps.getSHA.outputs.SHA}}
+    steps:
+      - name: Checkout SHA
+        id: getSHA
+        run: |
+          if ${{inputs.use_merge_sha}} == true; then
+            sha=$(echo ${{github.sha}})
+          else
+            sha=$(echo ${{ github.event.pull_request.head.sha }})
+          fi
+          echo "SHA: $sha"
+          echo "SHA=$sha" >> $GITHUB_OUTPUT
+
+  get_changed_files:
+    runs-on: ubuntu-latest
+    name: Get Changed files
+    outputs:
+      should_execute_tests: ${{ steps.processing.outputs.should_execute_tests }}
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 1
+      - id: files
+        uses: Ana06/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          format: 'json'
+      - id: processing
+        run: |
+          readarray -t modified_files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.modified }}')"
+          echo "Modified files: $modified_files"
+          found=false
+          for modified_file in "${modified_files[@]}"; do
+            if [[ "$modified_file" == "Dockerfile" || "$modified_file" == "docker-compose.yml" || "$modified_file" == "Dockerfile" || "$modified_file" == "Dockerfile-ut-func" ]]; then
+              found=true
+              break
+            fi
+          done
+          echo "Match Found: $found"
+          echo "::set-output name=should_execute_tests::$found"
+
   build-transformer-image-arm64:
     name: Build Transformer Docker Image ARM64
     runs-on: [self-hosted, Linux, ARM64]
+    needs: [get_sha, get_changed_files]
     steps:
       - name: Checkout
         uses: actions/[email protected]
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
 
       - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3.4.0
+        uses: docker/setup-buildx-action@v3.6.1
 
       - name: Login to DockerHub
-        uses: docker/login-action@v3.2.0
+        uses: docker/login-action@v3.3.0
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
       - name: Build Docker Image
-        uses: docker/build-push-action@v6.4.1
+        uses: docker/build-push-action@v6.5.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -62,12 +118,13 @@ jobs:
           # cache-to: type=gha,mode=max
 
       - name: Run Tests
+        if: ${{ inputs.skip_tests != true || needs.get_changed_files.outputs.should_execute_tests == true }}
         run: |
           docker run ${{ inputs.build_tag }} npm run test:js:ci
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
 
       - name: Build and Push Multi-platform Images
-        uses: docker/build-push-action@v6.4.1
+        uses: docker/build-push-action@v6.5.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -85,24 +142,25 @@ jobs:
   build-transformer-image-amd64:
     name: Build Transformer Docker Image AMD64
     runs-on: [self-hosted, Linux, X64]
+    needs: [get_sha, get_changed_files]
     steps:
       - name: Checkout
         uses: actions/[email protected]
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
 
       - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3.4.0
+        uses: docker/setup-buildx-action@v3.6.1
 
       - name: Login to DockerHub
-        uses: docker/login-action@v3.2.0
+        uses: docker/login-action@v3.3.0
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
       - name: Build Docker Image
-        uses: docker/build-push-action@v6.4.1
+        uses: docker/build-push-action@v6.5.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -113,12 +171,13 @@ jobs:
           # cache-to: type=gha,mode=max
 
       - name: Run Tests
+        if: ${{ inputs.skip_tests != true || needs.get_changed_files.outputs.should_execute_tests == true }}
         run: |
           docker run ${{ inputs.build_tag }} npm run test:js:ci
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
 
       - name: Build and Push Multi-platform Images
-        uses: docker/build-push-action@v6.4.1
+        uses: docker/build-push-action@v6.5.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -140,10 +199,10 @@ jobs:
 
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.4.0
+        uses: docker/setup-buildx-action@v3.6.1
 
       - name: Login to DockerHub
-        uses: docker/login-action@v3.2.0
+        uses: docker/login-action@v3.3.0
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}

.github/workflows/dt-test-and-report-code-coverage.yml

@@ -13,10 +13,22 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  get_workflow_url:
+    runs-on: ubuntu-latest
+    steps:
+      - id: get_url
+        run: |
+          curl -s https://api.github.com/repos/${{ github.repository }}/actions/workflows/${{ github.workflow }}/runs/${{ github.run_id }} | jq -r .html_url >> workflow_url.txt
+          echo "::set-output name=workflow_url::$(cat workflow_url.txt)"
+    outputs:
+      url: ${{ steps.get_url.outputs.workflow_url }}
+
   coverage:
     name: Code Coverage
     runs-on: ubuntu-latest
-
+    needs: [get_workflow_url]
+    outputs:
+      tests_run_outcome: ${{steps.run_tests.outcome}}
     steps:
       - name: Checkout
         uses: actions/[email protected]
@@ -33,6 +45,8 @@ jobs:
         run: npm ci
 
       - name: Run Tests
+        id: run_tests
+        continue-on-error: true
         run: |
           # Supress logging in tests
           LOG_LEVEL=100 npm run test:js:ci
@@ -70,3 +84,13 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+  notify:
+    name: slack notification on failure
+    needs: [get_workflow_url, coverage]
+    if: needs.coverage.outputs.tests_run_outcome == 'failure' || failure()
+    uses: ./.github/workflows/slack-notify.yml
+    with:
+      workflow_url: ${{ needs.get_workflow_url.outputs.url }}
+      should_notify: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
+    secrets: inherit

.github/workflows/prepare-for-dev-deploy.yml

@@ -60,6 +60,7 @@ jobs:
       dockerfile: Dockerfile
       load_target: development
       push_target: production
+      use_merge_sha: true
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/prepare-for-prod-dt-deploy.yml

@@ -57,6 +57,8 @@ jobs:
       load_target: development
       push_target: production
       build_type: dt
+      use_merge_sha: true
+      skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/prepare-for-prod-ut-deploy.yml

@@ -60,6 +60,8 @@ jobs:
       load_target: development
       push_target: production
       build_type: ut
+      use_merge_sha: true
+      skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/prepare-for-staging-deploy.yml

@@ -52,6 +52,7 @@ jobs:
       dockerfile: Dockerfile
       load_target: development
       push_target: production
+      use_merge_sha: true
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/slack-notify.yml

@@ -0,0 +1,51 @@
+name: Notify workflow failure
+
+on:
+  workflow_call:
+    inputs:
+      should_notify:
+        type: boolean
+        default: true
+      workflow_url:
+        type: string
+        required: true
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.should_notify }}
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: notify
+        uses: slackapi/[email protected]
+        continue-on-error: true
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          PROJECT_NAME: 'Rudder Transformer'
+        with:
+          channel-id: ${{ secrets.SLACK_INTEGRATION_DEV_CHANNEL_ID }}
+          payload: |
+            {
+              "text": "*<prod release tests failed>*\nCC: <!subteam^S02AEQL26CT>",
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": ":siren2:  prod release tests - Failed  :siren2:"
+                  }
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "*<${{inputs.workflow_url}}|failed workflow>*\nCC: <!subteam^S02AEQL26CT>"
+                  }
+                }
+              ]
+            }

.github/workflows/verify.yml

@@ -28,13 +28,29 @@ jobs:
       - name: Install Dependencies
         run: npm ci
 
-      - name: Run Lint Checks
+      - id: files
+        uses: Ana06/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run format Checks
+        run: |
+          npx prettier ${{steps.files.outputs.added_modified}} --write
+
+      - run: git diff --exit-code
+
+      - name: Formatting Error message
+        if: ${{ failure() }}
+        run: |
+          echo 'prettier formatting failure. Ensure you run `npm run format` and commit the files.'
+
+      - name: Run eslint Checks
         run: |
-          npm run lint
+          npx eslint ${{steps.files.outputs.added_modified}} --fix
 
       - run: git diff --exit-code
 
-      - name: Error message
+      - name: Eslint Error message
         if: ${{ failure() }}
         run: |
-          echo 'Eslint check is failing Ensure you have run `npm run lint` and committed the files locally.'
+          echo 'Eslint failure. Ensure you run `npm run lint:fix` and commit the files.'

.husky/commit-msg

@@ -1,4 +1,2 @@
-#!/usr/bin/env sh
-. "$(dirname -- "$0")/_/husky.sh"
 
 npm run commit-msg

.husky/pre-commit

@@ -1,4 +1,2 @@
-#!/usr/bin/env sh
-. "$(dirname -- "$0")/_/husky.sh"
 
 npm run pre-commit

.vscode/settings.json

@@ -13,6 +13,10 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.formatOnSave": true
   },
+  "[json]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.formatOnSave": true
+  },
   "[yaml]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.formatOnSave": true

go/webhook/testcases/testdata/testcases/close_crm/group_creation.json

@@ -34,7 +34,8 @@
             ]
           },
           "previous_data": {}
-        }
+        },
+        "source": {}
       },
       "headers": {
         "Content-Type": "application/json"

go/webhook/testcases/testdata/testcases/close_crm/lead_deletion.json

@@ -43,7 +43,8 @@
             "updated_by": "user_123",
             "created_by": "user_123"
           }
-        }
+        },
+        "source": {}
       },
       "headers": {
         "Content-Type": "application/json"

go/webhook/testcases/testdata/testcases/close_crm/lead_update.json

@@ -61,7 +61,8 @@
           "object_type": "opportunity",
           "lead_id": "lead_123"
         },
-        "subscription_id": "whsub_123"
+        "subscription_id": "whsub_123",
+        "source": {}
       },
       "headers": {
         "Content-Type": "application/json"