Merge branch 'develop' into chore.salesforce.update-proxy-tests.v1

.github/pull_request_template.md

@@ -26,6 +26,8 @@ N/A
 
 N/A
 
+@coderabbitai review
+
 <hr>
 
 ### Developer checklist

.github/workflows/component-test-report.yml

@@ -28,7 +28,7 @@ jobs:
           fetch-depth: 1
 
       - name: Setup Node
-        uses: actions/[email protected].1
+        uses: actions/[email protected].2
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
@@ -40,13 +40,11 @@ jobs:
         run: |
           npm run test:ts -- component
 
-
       - name: Uplaod Report to S3
         run: |
           aws s3 cp ./test_reports/ s3://test-integrations-dev/integrations-test-reports/rudder-transformer/${{ github.event.number }}/ --recursive
-
     
-      - name: Comment on PR with S3 Object URL
+      - name: Add Test Report Link as Comment on PR
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.PAT }}
@@ -55,10 +53,23 @@ jobs:
             // Get the pull request number
             const prNumber = context.payload.pull_request.number;
             const commentBody = `Test report for this run is available at: https://test-integrations-dev.s3.amazonaws.com/integrations-test-reports/rudder-transformer/${prNumber}/test-report.html`;
-            
+
+            // find all the comments of the PR
+            const issueComments = await github.paginate(github.rest.issues.listComments, {
+              owner,
+              repo,
+              issue_number: prNumber,
+            });
+
+            for (const comment of issueComments) {
+              if (comment.body === commentBody) {
+                console.log('Comment already exists');
+                return;
+              }
+            }
 
             // Comment on the pull request
-            github.rest.issues.createComment({
+            await github.rest.issues.createComment({
               owner,
               repo,
               issue_number: prNumber,

.github/workflows/create-hotfix-branch.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     # Only allow these users to create new hotfix branch from 'main'
-    if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'koladilip' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'ujjwal-ab') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' ||  github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'ujjwal-ab')
+    if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'koladilip' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'utsabc') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' ||  github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'utsabc')
     steps:
       - name: Create Branch
         uses: peterjgrainger/[email protected]

.github/workflows/draft-new-release.yml

@@ -8,15 +8,15 @@ jobs:
     runs-on: ubuntu-latest
 
     # Only allow release stakeholders to initiate releases
-    if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'koladilip' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823'  || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'ujjwal-ab') &&  (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'ujjwal-ab')
+    if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'koladilip' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823'  || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'utsabc') &&  (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'utsabc')
     steps:
       - name: Checkout
         uses: actions/[email protected]
         with:
           fetch-depth: 0
 
       - name: Setup Node
-        uses: actions/[email protected].1
+        uses: actions/[email protected].2
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'

.github/workflows/dt-test-and-report-code-coverage.yml

@@ -20,7 +20,7 @@ jobs:
           fetch-depth: 1
 
       - name: Setup Node
-        uses: actions/[email protected].1
+        uses: actions/[email protected].2
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
@@ -40,12 +40,14 @@ jobs:
           npm run lint:fix
 
       - name: Upload Coverage Reports to Codecov
-        uses: codecov/codecov-action@v3.1.4
+        uses: codecov/codecov-action@v4.0.1
         with:
           directory: ./reports/coverage
 
       - name: Upload TS Coverage Reports to Codecov
-        uses: codecov/[email protected]
+        uses: codecov/[email protected]
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           directory: ./reports/ts-coverage
 

.github/workflows/publish-new-release.yml

@@ -30,7 +30,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Node
-        uses: actions/[email protected].1
+        uses: actions/[email protected].2
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
@@ -89,7 +89,7 @@ jobs:
 
       - name: Notify Slack Channel
         id: slack
-        uses: slackapi/slack-github-action@v1.24.0
+        uses: slackapi/slack-github-action@v1.25.0
         continue-on-error: true
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/ut-tests.yml

@@ -26,7 +26,7 @@ jobs:
           fetch-depth: 1
 
       - name: Setup Node
-        uses: actions/[email protected].1
+        uses: actions/[email protected].2
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'

.gitignore

@@ -132,7 +132,7 @@ dist
 
 # Others
 **/.DS_Store
-
+.dccache
 
 .idea
 

CHANGELOG.md

@@ -2,6 +2,28 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.56.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.55.0...v1.56.0) (2024-02-19)
+
+
+### Features
+
+* **hs:** chunking data based on batch limit ([#2907](https://github.com/rudderlabs/rudder-transformer/issues/2907)) ([a60694c](https://github.com/rudderlabs/rudder-transformer/commit/a60694cef1da31d27a5cf90264548cad793f556f))
+* onboard bluecore integration ([#3061](https://github.com/rudderlabs/rudder-transformer/issues/3061)) ([aef5f8e](https://github.com/rudderlabs/rudder-transformer/commit/aef5f8e5f267262e0f9e10229f14f2bcc8ad29e2))
+* tiktok_offline_events added support for all Standard  events ([#3094](https://github.com/rudderlabs/rudder-transformer/issues/3094)) ([b5cdccb](https://github.com/rudderlabs/rudder-transformer/commit/b5cdccb75fe68150816140174087fddad677db10))
+
+
+### Bug Fixes
+
+* add support of placing properties at root in af ([#3082](https://github.com/rudderlabs/rudder-transformer/issues/3082)) ([0f01524](https://github.com/rudderlabs/rudder-transformer/commit/0f01524b6f4f2f82efc21f88f8c97cb6fdaf91ea))
+* amplitude batch output metadata ([#3077](https://github.com/rudderlabs/rudder-transformer/issues/3077)) ([69c8348](https://github.com/rudderlabs/rudder-transformer/commit/69c83489c85486c9b2aed4a1292bd9f0aae9ca44))
+* amplitude: Error handling for missing event type ([#3079](https://github.com/rudderlabs/rudder-transformer/issues/3079)) ([f7ec0a1](https://github.com/rudderlabs/rudder-transformer/commit/f7ec0a1244a7b97e6b40de5ed9881c63300866dc))
+* custify user-regulation logic ([#3076](https://github.com/rudderlabs/rudder-transformer/issues/3076)) ([9683161](https://github.com/rudderlabs/rudder-transformer/commit/9683161612c7e3b9c2be95a2728f68ec7dcf69f4))
+* error handling for auth0 source ([#3038](https://github.com/rudderlabs/rudder-transformer/issues/3038)) ([2a21274](https://github.com/rudderlabs/rudder-transformer/commit/2a21274333350c615991f7b56b81b766502d5bf4))
+* **ga4:** failures not considered with 200 status in events tab ([#3089](https://github.com/rudderlabs/rudder-transformer/issues/3089)) ([6a364fb](https://github.com/rudderlabs/rudder-transformer/commit/6a364fba34c46b15c0fe4b06ecfa6f4b81b6f436))
+* gaoc batching order ([#3064](https://github.com/rudderlabs/rudder-transformer/issues/3064)) ([a98cabd](https://github.com/rudderlabs/rudder-transformer/commit/a98cabdfe7781ada12baf742df4a3a439fc5fecd))
+* resolve bugsnag issue caused due to undefined properties ([#3086](https://github.com/rudderlabs/rudder-transformer/issues/3086)) ([d522b35](https://github.com/rudderlabs/rudder-transformer/commit/d522b35c908a9f262ba3ba27dda0ea5d9ac5bc6b))
+* tiktok ads v2 error handling ([#3084](https://github.com/rudderlabs/rudder-transformer/issues/3084)) ([b6edff4](https://github.com/rudderlabs/rudder-transformer/commit/b6edff46fa0e0e210e82206fea46a064e3fbe00f))
+
 ## [1.55.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.54.4...v1.55.0) (2024-02-05)
 
 

README.md

@@ -1,3 +1,8 @@
+<p align="center">
+⚠️ Docker image for rudder-transformer has been moved to new org <a href="https://hub.docker.com/r/rudderstack/rudder-transformer/tags">rudderstack/rudder-transformer</a>
+  <br/><br/>
+ </p>
+
 [![codecov](https://codecov.io/gh/rudderlabs/rudder-transformer/branch/develop/graph/badge.svg?token=G24OON85SB)](https://codecov.io/gh/rudderlabs/rudder-transformer)
 
 # RudderStack Transformer

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.55.0",
+  "version": "1.56.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -65,7 +65,7 @@
     "@ndhoule/extend": "^2.0.0",
     "@pyroscope/nodejs": "^0.2.6",
     "@rudderstack/integrations-lib": "^0.2.2",
-    "@rudderstack/workflow-engine": "^0.6.9",
+    "@rudderstack/workflow-engine": "^0.7.2",
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",
     "ajv-formats": "^2.1.1",

src/adapters/network.js

@@ -49,18 +49,24 @@ const fireHTTPStats = (clientResponse, startTime, statTags) => {
   const destType = statTags.destType ? statTags.destType : '';
   const feature = statTags.feature ? statTags.feature : '';
   const endpointPath = statTags.endpointPath ? statTags.endpointPath : '';
+  const requestMethod = statTags.requestMethod ? statTags.requestMethod : '';
+  const module = statTags.module ? statTags.module : '';
   const statusCode = clientResponse.success ? clientResponse.response.status : '';
   stats.timing('outgoing_request_latency', startTime, {
     feature,
     destType,
     endpointPath,
+    requestMethod,
+    module
   });
   stats.counter('outgoing_request_count', 1, {
     feature,
     destType,
     endpointPath,
     success: clientResponse.success,
     statusCode,
+    requestMethod,
+    module
   });
 };
 

src/cdk/v2/destinations/bluecore/config.js

@@ -0,0 +1,56 @@
+const { getMappingConfig } = require('../../../../v0/util');
+
+const BASE_URL = 'https://api.bluecore.com/api/track/mobile/v1';
+
+const CONFIG_CATEGORIES = {
+  IDENTIFY: {
+    name: 'bluecoreIdentifyConfig',
+    type: 'identify',
+  },
+  TRACK: {
+    name: 'bluecoreTrackConfig',
+    type: 'track',
+  },
+  COMMON: {
+    name: 'bluecoreCommonConfig',
+    type: 'common',
+  },
+};
+
+const EVENT_NAME_MAPPING = [
+  {
+    src: ['product viewed'],
+    dest: 'viewed_product',
+  },
+  {
+    src: ['products searched'],
+    dest: 'search',
+  },
+  {
+    src: ['product added'],
+    dest: 'add_to_cart',
+  },
+  {
+    src: ['product removed'],
+    dest: 'remove_from_cart',
+  },
+  {
+    src: ['product added to wishlist'],
+    dest: 'wishlist',
+  },
+  {
+    src: ['order completed'],
+    dest: 'purchase',
+  },
+];
+
+const BLUECORE_EXCLUSION_FIELDS = ['query', 'order_id', 'total'];
+
+const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname);
+module.exports = {
+  CONFIG_CATEGORIES,
+  MAPPING_CONFIG,
+  EVENT_NAME_MAPPING,
+  BASE_URL,
+  BLUECORE_EXCLUSION_FIELDS,
+};

src/cdk/v2/destinations/bluecore/data/bluecoreCommonConfig.json

@@ -0,0 +1,52 @@
+[
+  {
+    "destKey": "properties.customer.name",
+    "sourceKeys": "name",
+    "required": false,
+    "sourceFromGenericMap": true
+  },
+  {
+    "destKey": "properties.customer.first_name",
+    "sourceKeys": "firstName",
+    "required": false,
+    "sourceFromGenericMap": true
+  },
+  {
+    "destKey": "properties.customer.last_name",
+    "sourceKeys": "lastName",
+    "required": false,
+    "sourceFromGenericMap": true
+  },
+  {
+    "destKey": "properties.customer.age",
+    "sourceKeys": ["context.traits.age", "traits.age"],
+    "required": false
+  },
+  {
+    "destKey": "properties.customer.sex",
+    "sourceKeys": ["traits.gender", "context.traits.gender", "traits.sex", "context.traits.sex"],
+    "required": false
+  },
+  {
+    "destKey": "properties.customer.address",
+    "sourceKeys": "address",
+    "required": false,
+    "sourceFromGenericMap": true
+  },
+  {
+    "destKey": "properties.customer.email",
+    "sourceKeys": "email",
+    "required": false,
+    "sourceFromGenericMap": true
+  },
+  {
+    "destKey": "properties.client",
+    "sourceKeys": "context.app.version",
+    "required": false
+  },
+  {
+    "destKey": "properties.device",
+    "sourceKeys": "context.device.model",
+    "required": false
+  }
+]

src/cdk/v2/destinations/bluecore/data/bluecoreIdentifyConfig.json

@@ -0,0 +1,7 @@
+[
+  {
+    "destKey": "event",
+    "sourceKeys": ["traits.action", "context.traits.action"],
+    "required": false
+  }
+]