Merge branch 'develop' into feat.logging-tranformer-improvement

.eslintrc.json

@@ -9,9 +9,9 @@
     "airbnb-base",
     "airbnb-typescript/base",
     "plugin:sonarjs/recommended",
-    "prettier",
     "plugin:json/recommended",
-    "plugin:@typescript-eslint/recommended"
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
   ],
   "plugins": ["@typescript-eslint", "unicorn"],
   "globals": {},

.github/dependabot.yml

@@ -4,3 +4,11 @@ updates:
     directory: '/'
     schedule:
       interval: 'weekly'
+  - package-ecosystem: 'docker'
+    directory: '/'
+    schedule:
+      interval: 'daily'
+  - package-ecosystem: 'npm'
+    directory: '/'
+    schedule:
+      interval: 'daily'

.github/pull_request_template.md

@@ -26,6 +26,8 @@ N/A
 
 N/A
 
+@coderabbitai review
+
 <hr>
 
 ### Developer checklist

.github/workflows/build-push-docker-image.yml

@@ -155,7 +155,7 @@ jobs:
         if: ${{ inputs.build_type == 'dt' }}
         run: |
           docker buildx imagetools create -t rudderstack/rudder-transformer:latest ${{ inputs.push_tags }}-amd64 ${{ inputs.push_tags }}-arm64
-      
+
       - name: Create latest ut multi-arch manifest
         # To be triggered only for release/hotfix PR merges coming from `prepare-for-prod-ut-deploy.yaml`
         if: ${{ inputs.build_type == 'ut' }}

.github/workflows/commitlint.yml

@@ -0,0 +1,36 @@
+name: Commitlint
+
+on: [push]
+
+jobs:
+  commitlint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Print versions
+        run: |
+          git --version
+          node --version
+          npm --version
+          npx commitlint --version
+
+      # Run the commitlint action, considering its own dependencies and yours as well 🚀
+      # `github.workspace` is the path to your repository.
+      - uses: wagoid/commitlint-github-action@v5
+        env:
+          NODE_PATH: ${{ github.workspace }}/node_modules
+        with:
+          commitDepth: 1

.github/workflows/component-test-report.yml

@@ -28,7 +28,7 @@ jobs:
           fetch-depth: 1
 
       - name: Setup Node
-        uses: actions/[email protected].1
+        uses: actions/[email protected].2
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
@@ -40,13 +40,11 @@ jobs:
         run: |
           npm run test:ts -- component
 
-
       - name: Uplaod Report to S3
         run: |
           aws s3 cp ./test_reports/ s3://test-integrations-dev/integrations-test-reports/rudder-transformer/${{ github.event.number }}/ --recursive
 
-    
-      - name: Comment on PR with S3 Object URL
+      - name: Add Test Report Link as Comment on PR
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.PAT }}
@@ -55,14 +53,25 @@ jobs:
             // Get the pull request number
             const prNumber = context.payload.pull_request.number;
             const commentBody = `Test report for this run is available at: https://test-integrations-dev.s3.amazonaws.com/integrations-test-reports/rudder-transformer/${prNumber}/test-report.html`;
-            
+
+            // find all the comments of the PR
+            const issueComments = await github.paginate(github.rest.issues.listComments, {
+              owner,
+              repo,
+              issue_number: prNumber,
+            });
+
+            for (const comment of issueComments) {
+              if (comment.body === commentBody) {
+                console.log('Comment already exists');
+                return;
+              }
+            }
 
             // Comment on the pull request
-            github.rest.issues.createComment({
+            await github.rest.issues.createComment({
               owner,
               repo,
               issue_number: prNumber,
               body: commentBody
             });
-
-        

.github/workflows/create-hotfix-branch.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     # Only allow these users to create new hotfix branch from 'main'
-    if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'koladilip' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'ujjwal-ab') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' ||  github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'ujjwal-ab')
+    if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'koladilip' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'utsabc') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' ||  github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'utsabc')
     steps:
       - name: Create Branch
         uses: peterjgrainger/[email protected]

.github/workflows/draft-new-release.yml

@@ -8,15 +8,15 @@ jobs:
     runs-on: ubuntu-latest
 
     # Only allow release stakeholders to initiate releases
-    if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'koladilip' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823'  || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'ujjwal-ab') &&  (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'ujjwal-ab')
+    if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'koladilip' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823'  || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'utsabc') &&  (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'utsabc')
     steps:
       - name: Checkout
         uses: actions/[email protected]
         with:
           fetch-depth: 0
 
       - name: Setup Node
-        uses: actions/[email protected].1
+        uses: actions/[email protected].2
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'

.github/workflows/dt-test-and-report-code-coverage.yml

@@ -20,7 +20,7 @@ jobs:
           fetch-depth: 1
 
       - name: Setup Node
-        uses: actions/[email protected].1
+        uses: actions/[email protected].2
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
@@ -40,12 +40,16 @@ jobs:
           npm run lint:fix
 
       - name: Upload Coverage Reports to Codecov
-        uses: codecov/[email protected]
+        uses: codecov/[email protected]
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           directory: ./reports/coverage
 
       - name: Upload TS Coverage Reports to Codecov
-        uses: codecov/[email protected]
+        uses: codecov/[email protected]
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           directory: ./reports/ts-coverage
 

.github/workflows/prepare-for-prod-dt-deploy.yml

@@ -144,3 +144,39 @@ jobs:
           git push -u origin hosted-transformer-$TAG_NAME
 
           gh pr create --fill
+
+      - name: Update helm charts and raise pull request for enterprise customers on dedicated transformers
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+        run: |
+          cd rudder-devops
+          git checkout -b dedicated-transformer-$TAG_NAME
+
+          cd customer-objects
+
+          declare -a enabled_ut_customers=()
+          declare -a sub_directories=('enterprise-us' 'enterprise-eu')
+
+          # identify the customers enabled in sub-directories
+          for directory in "${sub_directories[@]}"; do
+            for f in "./$directory"/*; do
+              [[ -f $f ]] || continue
+
+              enabled="$(yq e '.spec.user_transformer.enabled' $f)"
+              if [ $enabled == "true" ]; then
+                  enabled_ut_customers+=( $f )
+              fi
+            done
+          done
+
+          # bump up the customers version and repository information
+          for customer in "${enabled_ut_customers[@]}"; do
+            yq eval -i ".spec.user_transformer.image.version=\"$TAG_NAME\"" $customer
+            yq eval -i ".spec.user_transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" $customer
+            git add $customer
+          done
+
+          git commit -m "chore: upgrade dedicated transformers to $TAG_NAME"
+          git push -u origin dedicated-transformer-$TAG_NAME
+
+          gh pr create --fill

.github/workflows/prepare-for-prod-rollback.yml

@@ -27,11 +27,14 @@ jobs:
           git config --global user.name "GitHub Actions"
           git config --global user.email "[email protected]"
 
+      - name: Clone Devops Repo
+        run: |
+          git clone https://${{secrets.PAT}}@github.com/rudderlabs/rudder-devops.git
+
       - name: Update Helm Charts and Raise Pull Request
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
         run: |
-          git clone https://${{secrets.PAT}}@github.com/rudderlabs/rudder-devops.git
           cd rudder-devops
           git checkout -b shared-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
 
@@ -57,3 +60,38 @@ jobs:
           git push -u origin shared-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
 
           gh pr create --fill
+
+      - name: Update helm charts and raise pull request for enterprise customers on dedicated transformers
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+        run: |
+          cd rudder-devops
+          git checkout -b dedicated-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
+
+          cd customer-objects
+
+          declare -a enabled_ut_customers=()
+          declare -a sub_directories=('enterprise-us' 'enterprise-eu')
+
+          # identify the customers enabled in sub-directories
+          for directory in "${sub_directories[@]}"; do
+            for f in "./$directory"/*; do
+              [[ -f $f ]] || continue
+
+              enabled="$(yq e '.spec.user_transformer.enabled' $f)"
+              if [ $enabled == "true" ]; then
+                  enabled_ut_customers+=( $f )
+              fi
+            done
+          done
+
+          # bump up the customers version and repository information
+          for customer in "${enabled_ut_customers[@]}"; do
+            yq eval -i ".spec.user_transformer.image.version=\"${{ steps.target-version.outputs.tag_name }}\"" $customer
+            git add $customer
+          done
+
+          git commit -m "chore: rollback dedicated transformers to ${{ steps.target-version.outputs.tag_name }}"
+          git push -u origin dedicated-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
+
+          gh pr create --fill

.github/workflows/prepare-for-staging-deploy.yml

@@ -68,7 +68,6 @@ jobs:
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
-
   create-pull-request:
     name: Update Helm Charts For Staging and Create Pull Request
     runs-on: ubuntu-latest
@@ -102,7 +101,7 @@ jobs:
           cd rudder-devops
           BRANCH_NAME="shared-transformer-$TAG_NAME"
           echo $BRANCH_NAME
-          if [ `git ls-remote --heads origin $BRANCH_NAME 2>/dev/null` ]
+          if [ -n "$(git ls-remote --heads origin $BRANCH_NAME 2>/dev/null)" ]
           then
             echo "Staging deployment branch already exists!"
           else
@@ -113,9 +112,9 @@ jobs:
             yq eval -i ".user-transformer.image.tag=\"$TAG_NAME\"" staging.yaml
             git add staging.yaml
 
-            cd ../../../../config-be-rudder-transformer
-            yq eval -i ".config-be-rudder-transformer.image.tag=\"$TAG_NAME\"" values.staging.yaml
-            yq eval -i ".config-be-user-transformer.image.tag=\"$TAG_NAME\"" values.staging.yaml
+            cd ../../../../config-be-rudder-transformer/environment/staging
+            yq eval -i ".config-be-rudder-transformer.image.tag=\"$TAG_NAME\"" base.yaml
+            yq eval -i ".config-be-user-transformer.image.tag=\"$TAG_NAME\"" base.yaml
             git add values.staging.yaml
 
             git commit -m "chore: upgrade staging env transformers to \"$TAG_NAME\""

.github/workflows/publish-new-release.yml

@@ -30,7 +30,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Node
-        uses: actions/[email protected].1
+        uses: actions/[email protected].2
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
@@ -89,7 +89,7 @@ jobs:
 
       - name: Notify Slack Channel
         id: slack
-        uses: slackapi/slack-github-action@v1.24.0
+        uses: slackapi/slack-github-action@v1.25.0
         continue-on-error: true
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/ut-tests.yml

@@ -26,7 +26,7 @@ jobs:
           fetch-depth: 1
 
       - name: Setup Node
-        uses: actions/[email protected].1
+        uses: actions/[email protected].2
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'

.github/workflows/verify.yml

@@ -0,0 +1,36 @@
+name: Verify
+
+on:
+  pull_request:
+
+jobs:
+  formatting-lint:
+    name: Check for formatting & lint errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          # Make sure the actual branch is checked out when running on pull requests
+          ref: ${{ github.head_ref }}
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version-file: .nvmrc
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Run Lint Checks
+        run: |
+          npm run lint
+
+      - run: git diff --exit-code
+
+      - name: Error message
+        if: ${{ failure() }}
+        run: |
+          echo 'Eslint check is failing Ensure you have run `npm run lint` and committed the files locally.'

.gitignore

@@ -137,4 +137,5 @@ dist
 .idea
 
 # component test report
-test_reports/
+test_reports/
+temp/

.nvmrc

@@ -1 +1 @@
-18.19.0
+18.19.1

.prettierignore

@@ -7,3 +7,4 @@ test/**/*.js
 !test/**/data.js
 src/util/lodash-es-core.js
 src/util/url-search-params.min.js
+dist