Merge branch 'develop' of github.com:rudderlabs/rudder-transformer in…

…to fix.throttle-errors

.eslintrc.json

@@ -9,9 +9,9 @@
     "airbnb-base",
     "airbnb-typescript/base",
     "plugin:sonarjs/recommended",
-    "prettier",
     "plugin:json/recommended",
-    "plugin:@typescript-eslint/recommended"
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
   ],
   "plugins": ["@typescript-eslint", "unicorn"],
   "globals": {},

.github/workflows/build-push-docker-image.yml

@@ -155,7 +155,7 @@ jobs:
         if: ${{ inputs.build_type == 'dt' }}
         run: |
           docker buildx imagetools create -t rudderstack/rudder-transformer:latest ${{ inputs.push_tags }}-amd64 ${{ inputs.push_tags }}-arm64
-      
+
       - name: Create latest ut multi-arch manifest
         # To be triggered only for release/hotfix PR merges coming from `prepare-for-prod-ut-deploy.yaml`
         if: ${{ inputs.build_type == 'ut' }}

.github/workflows/commitlint.yml

@@ -0,0 +1,36 @@
+name: Commitlint
+
+on: [push]
+
+jobs:
+  commitlint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Print versions
+        run: |
+          git --version
+          node --version
+          npm --version
+          npx commitlint --version
+
+      # Run the commitlint action, considering its own dependencies and yours as well 🚀
+      # `github.workspace` is the path to your repository.
+      - uses: wagoid/commitlint-github-action@v5
+        env:
+          NODE_PATH: ${{ github.workspace }}/node_modules
+        with:
+          commitDepth: 1

.github/workflows/component-test-report.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Uplaod Report to S3
         run: |
           aws s3 cp ./test_reports/ s3://test-integrations-dev/integrations-test-reports/rudder-transformer/${{ github.event.number }}/ --recursive
-    
+
       - name: Add Test Report Link as Comment on PR
         uses: actions/github-script@v7
         with:
@@ -75,5 +75,3 @@ jobs:
               issue_number: prNumber,
               body: commentBody
             });
-
-        

.github/workflows/dt-test-and-report-code-coverage.yml

@@ -41,6 +41,8 @@ jobs:
 
       - name: Upload Coverage Reports to Codecov
         uses: codecov/[email protected]
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           directory: ./reports/coverage
 

.github/workflows/prepare-for-prod-dt-deploy.yml

@@ -144,3 +144,39 @@ jobs:
           git push -u origin hosted-transformer-$TAG_NAME
 
           gh pr create --fill
+
+      - name: Update helm charts and raise pull request for enterprise customers on dedicated transformers
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+        run: |
+          cd rudder-devops
+          git checkout -b dedicated-transformer-$TAG_NAME
+
+          cd customer-objects
+
+          declare -a enabled_ut_customers=()
+          declare -a sub_directories=('enterprise-us' 'enterprise-eu')
+
+          # identify the customers enabled in sub-directories
+          for directory in "${sub_directories[@]}"; do
+            for f in "./$directory"/*; do
+              [[ -f $f ]] || continue
+
+              enabled="$(yq e '.spec.user_transformer.enabled' $f)"
+              if [ $enabled == "true" ]; then
+                  enabled_ut_customers+=( $f )
+              fi
+            done
+          done
+
+          # bump up the customers version and repository information
+          for customer in "${enabled_ut_customers[@]}"; do
+            yq eval -i ".spec.user_transformer.image.version=\"$TAG_NAME\"" $customer
+            yq eval -i ".spec.user_transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" $customer
+            git add $customer
+          done
+
+          git commit -m "chore: upgrade dedicated transformers to $TAG_NAME"
+          git push -u origin dedicated-transformer-$TAG_NAME
+
+          gh pr create --fill

.github/workflows/prepare-for-prod-rollback.yml

@@ -27,11 +27,14 @@ jobs:
           git config --global user.name "GitHub Actions"
           git config --global user.email "[email protected]"
 
+      - name: Clone Devops Repo
+        run: |
+          git clone https://${{secrets.PAT}}@github.com/rudderlabs/rudder-devops.git
+
       - name: Update Helm Charts and Raise Pull Request
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
         run: |
-          git clone https://${{secrets.PAT}}@github.com/rudderlabs/rudder-devops.git
           cd rudder-devops
           git checkout -b shared-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
 
@@ -57,3 +60,38 @@ jobs:
           git push -u origin shared-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
 
           gh pr create --fill
+
+      - name: Update helm charts and raise pull request for enterprise customers on dedicated transformers
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+        run: |
+          cd rudder-devops
+          git checkout -b dedicated-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
+
+          cd customer-objects
+
+          declare -a enabled_ut_customers=()
+          declare -a sub_directories=('enterprise-us' 'enterprise-eu')
+
+          # identify the customers enabled in sub-directories
+          for directory in "${sub_directories[@]}"; do
+            for f in "./$directory"/*; do
+              [[ -f $f ]] || continue
+
+              enabled="$(yq e '.spec.user_transformer.enabled' $f)"
+              if [ $enabled == "true" ]; then
+                  enabled_ut_customers+=( $f )
+              fi
+            done
+          done
+
+          # bump up the customers version and repository information
+          for customer in "${enabled_ut_customers[@]}"; do
+            yq eval -i ".spec.user_transformer.image.version=\"${{ steps.target-version.outputs.tag_name }}\"" $customer
+            git add $customer
+          done
+
+          git commit -m "chore: rollback dedicated transformers to ${{ steps.target-version.outputs.tag_name }}"
+          git push -u origin dedicated-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
+
+          gh pr create --fill

.github/workflows/prepare-for-staging-deploy.yml

@@ -68,7 +68,6 @@ jobs:
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
-
   create-pull-request:
     name: Update Helm Charts For Staging and Create Pull Request
     runs-on: ubuntu-latest
@@ -102,7 +101,7 @@ jobs:
           cd rudder-devops
           BRANCH_NAME="shared-transformer-$TAG_NAME"
           echo $BRANCH_NAME
-          if [ `git ls-remote --heads origin $BRANCH_NAME 2>/dev/null` ]
+          if [ -n "$(git ls-remote --heads origin $BRANCH_NAME 2>/dev/null)" ]
           then
             echo "Staging deployment branch already exists!"
           else

.github/workflows/verify.yml

@@ -0,0 +1,36 @@
+name: Verify
+
+on:
+  pull_request:
+
+jobs:
+  formatting-lint:
+    name: Check for formatting & lint errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          # Make sure the actual branch is checked out when running on pull requests
+          ref: ${{ github.head_ref }}
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version-file: .nvmrc
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Run Lint Checks
+        run: |
+          npm run lint
+
+      - run: git diff --exit-code
+
+      - name: Error message
+        if: ${{ failure() }}
+        run: |
+          echo 'Eslint check is failing Ensure you have run `npm run lint` and committed the files locally.'

.gitignore

@@ -137,4 +137,5 @@ dist
 .idea
 
 # component test report
-test_reports/
+test_reports/
+temp/

.nvmrc

@@ -1 +1 @@
-18.19.0
+18.19.1

.prettierignore

@@ -7,3 +7,4 @@ test/**/*.js
 !test/**/data.js
 src/util/lodash-es-core.js
 src/util/url-search-params.min.js
+dist

.vscode/settings.json

@@ -0,0 +1,24 @@
+{
+  "prettier.requireConfig": true,
+  "prettier.configPath": ".prettierrc",
+  "[typescript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.formatOnSave": true
+  },
+  "[javascript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.formatOnSave": true
+  },
+  "[jsonc]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.formatOnSave": true
+  },
+  "[yaml]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.formatOnSave": true
+  },
+  "editor.codeActionsOnSave": {
+    "source.organizeImports": "never"
+  },
+  "eslint.validate": ["javascript", "typescript"]
+}

CHANGELOG.md

@@ -2,6 +2,59 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.58.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.58.0) (2024-03-04)
+
+
+### Features
+
+* add support for interaction events in sfmc ([#3109](https://github.com/rudderlabs/rudder-transformer/issues/3109)) ([0486049](https://github.com/rudderlabs/rudder-transformer/commit/0486049ba2ad96b50d8f29e96b46b96a8a5c9f76))
+* add support of custom page/screen event name in mixpanel ([#3098](https://github.com/rudderlabs/rudder-transformer/issues/3098)) ([0eb2393](https://github.com/rudderlabs/rudder-transformer/commit/0eb2393939fba2452ef7f07a1d149d87f18290c3))
+* consent mode support for google adwords remarketing list  ([#3143](https://github.com/rudderlabs/rudder-transformer/issues/3143)) ([7532c90](https://github.com/rudderlabs/rudder-transformer/commit/7532c90d7e1feac00f12961c56da18757010f44a))
+* **facebook:** update content_type mapping logic for fb pixel and fb conversions ([#3113](https://github.com/rudderlabs/rudder-transformer/issues/3113)) ([aea417c](https://github.com/rudderlabs/rudder-transformer/commit/aea417cd2691547399010c034cadbc5db6b0c6ee))
+* klaviyo profile mapping ([#3105](https://github.com/rudderlabs/rudder-transformer/issues/3105)) ([2761786](https://github.com/rudderlabs/rudder-transformer/commit/2761786ff3fc99ed6d4d3b7a6c2400226b1cfb12))
+* onboard new destination ninetailed ([#3106](https://github.com/rudderlabs/rudder-transformer/issues/3106)) ([0e2588e](https://github.com/rudderlabs/rudder-transformer/commit/0e2588ecd87f3b2c6877a099aa1cbf2d5325966c))
+
+
+### Bug Fixes
+
+* add error handling for tiktok ads ([#3144](https://github.com/rudderlabs/rudder-transformer/issues/3144)) ([e93e47f](https://github.com/rudderlabs/rudder-transformer/commit/e93e47f33e098104fb532916932fe38bbfeaa4a1))
+* **algolia:** added check for objectIds or filters to be non empty ([#3126](https://github.com/rudderlabs/rudder-transformer/issues/3126)) ([d619c97](https://github.com/rudderlabs/rudder-transformer/commit/d619c9769cd270cb2d16dad0865683ff4beb2d19))
+* clevertap remove stringification of array object properties ([#3048](https://github.com/rudderlabs/rudder-transformer/issues/3048)) ([69e43b6](https://github.com/rudderlabs/rudder-transformer/commit/69e43b6ffadeaec87b7440da34a341890ceba252))
+* convert to string from null in hs ([#3136](https://github.com/rudderlabs/rudder-transformer/issues/3136)) ([75e9f46](https://github.com/rudderlabs/rudder-transformer/commit/75e9f462b0ff9b9a8abab3c78dc7d147926e9e5e))
+* event fix and added utility ([#3142](https://github.com/rudderlabs/rudder-transformer/issues/3142)) ([9b705b7](https://github.com/rudderlabs/rudder-transformer/commit/9b705b71a9d3a595ea0fbf532602c3941b0a18db))
+* metadata structure correction ([#3119](https://github.com/rudderlabs/rudder-transformer/issues/3119)) ([8351b5c](https://github.com/rudderlabs/rudder-transformer/commit/8351b5cbbf81bbc14b2f884feaae4ad3ca59a39a))
+* one_signal:  Encode external_id in endpoint ([#3140](https://github.com/rudderlabs/rudder-transformer/issues/3140)) ([8a20886](https://github.com/rudderlabs/rudder-transformer/commit/8a2088608d6da4b35bbb506db2fc3df1e4d41f3b))
+* rakuten: sync property mapping sourcekeys to rudderstack standard spec ([#3129](https://github.com/rudderlabs/rudder-transformer/issues/3129)) ([2ebff95](https://github.com/rudderlabs/rudder-transformer/commit/2ebff956ff2aa74b008a8de832a31d8774d2d47e))
+* reddit revenue mapping for floating point values ([#3118](https://github.com/rudderlabs/rudder-transformer/issues/3118)) ([41f4078](https://github.com/rudderlabs/rudder-transformer/commit/41f4078011ef54334bb9ecc11a7b2ccc8831a4aa))
+* version deprecation failure false positive ([#3104](https://github.com/rudderlabs/rudder-transformer/issues/3104)) ([657b780](https://github.com/rudderlabs/rudder-transformer/commit/657b7805eb01da25a007d978198d5debf03917fd))
+
+### [1.57.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.0...v1.57.1) (2024-03-04)
+
+
+### Bug Fixes
+
+* amplitude fix for user operations ([7f2364e](https://github.com/rudderlabs/rudder-transformer/commit/7f2364e41167611c41003559de65cee1fece5464))
+* amplitude fix for user operations ([#3153](https://github.com/rudderlabs/rudder-transformer/issues/3153)) ([31869fb](https://github.com/rudderlabs/rudder-transformer/commit/31869fb114bb141d545de01c56f57b97e5aa54a6))
+
+## [1.57.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.56.1...v1.57.0) (2024-02-29)
+
+
+### Features
+
+* add event mapping support for branch destination ([#3135](https://github.com/rudderlabs/rudder-transformer/issues/3135)) ([cc94bba](https://github.com/rudderlabs/rudder-transformer/commit/cc94bba682f667877a721f63627adc6ff6a7386a))
+
+
+### Bug Fixes
+
+* marketo bulk upload zero and null value allowed ([#3134](https://github.com/rudderlabs/rudder-transformer/issues/3134)) ([4dcbf8f](https://github.com/rudderlabs/rudder-transformer/commit/4dcbf8fb189a39bb40b950742425a0b9da2d8d7c))
+
+### [1.56.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.56.0...v1.56.1) (2024-02-21)
+
+
+### Bug Fixes
+
+* update proxy data type for response handler input ([#3030](https://github.com/rudderlabs/rudder-transformer/issues/3030)) ([457a18b](https://github.com/rudderlabs/rudder-transformer/commit/457a18b2aec03aa0dfafcadc611b5f7176e97beb))
+
 ## [1.56.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.55.0...v1.56.0) (2024-02-19)
 
 

CONTRIBUTING.md

@@ -35,7 +35,6 @@ See the project's [README](README.md) for further information about working in t
    - Include instructions on how to test your changes.
 3. Your branch may be merged once all configured checks pass, including:
    - A review from appropriate maintainers
-4. Along with the PR in transformer raise a PR against [config-generator][config-generator] with the configurations.
 
 ## Committing
 

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.4
-FROM node:18.19.0-alpine3.18 AS base
+FROM node:18.19.1-alpine3.18 AS base
 ENV HUSKY 0
 
 RUN apk update

github-release.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  gitRawCommitsOpts: {
+    merges: null,
+  },
+};