Merge branch 'develop' into feat.intercom

.github/workflows/component-test-report.yml

@@ -0,0 +1,68 @@
+name: Component Test Reporter
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+permissions:
+  id-token: write # allows the JWT to be requested from GitHub's OIDC provider
+  contents: read # This is required for actions/checkout
+
+jobs:
+  test_and_upload:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_DEV_ACCOUNT_ID }}:role/${{ secrets.AWS_DEV_S3_SYNC_ROLE }}
+          aws-region: us-east-1
+
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 1
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Run Tests and Generate Report
+        run: |
+          npm run test:ts -- component
+
+
+      - name: Uplaod Report to S3
+        run: |
+          aws s3 cp ./test_reports/ s3://test-integrations-dev/integrations-test-reports/rudder-transformer/${{ github.event.number }}/ --recursive
+
+    
+      - name: Comment on PR with S3 Object URL
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.PAT }}
+          script: |
+            const { owner, repo } = context.repo;
+            // Get the pull request number
+            const prNumber = context.payload.pull_request.number;
+            const commentBody = `Test report for this run is available at: https://test-integrations-dev.s3.amazonaws.com/integrations-test-reports/rudder-transformer/${prNumber}/test-report.html`;
+            
+
+            // Comment on the pull request
+            github.rest.issues.createComment({
+              owner,
+              repo,
+              issue_number: prNumber,
+              body: commentBody
+            });
+
+        

.github/workflows/prepare-for-dev-deploy.yml

@@ -14,7 +14,7 @@ jobs:
   report-coverage:
     name: Report Code Coverage
     if: github.event_name == 'push'
-    uses: ./.github/workflows/report-code-coverage.yml
+    uses: ./.github/workflows/dt-test-and-report-code-coverage.yml
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 

.github/workflows/prepare-for-prod-dt-deploy.yml

@@ -14,7 +14,7 @@ jobs:
   report-coverage:
     name: Report Code Coverage
     if: github.event_name == 'push'
-    uses: ./.github/workflows/report-code-coverage.yml
+    uses: ./.github/workflows/dt-test-and-report-code-coverage.yml
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 

.github/workflows/prepare-for-prod-ut-deploy.yml

@@ -14,7 +14,7 @@ jobs:
   report-coverage:
     name: Report Code Coverage
     if: github.event_name == 'push'
-    uses: ./.github/workflows/report-code-coverage.yml
+    uses: ./.github/workflows/dt-test-and-report-code-coverage.yml
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 

.github/workflows/ut-tests.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Go
-        uses: actions/setup-go@v4.1.0
+        uses: actions/setup-go@v5.0.0
         with:
           go-version: 1.17
 

.gitignore

@@ -134,4 +134,7 @@ dist
 **/.DS_Store
 
 
-.idea
+.idea
+
+# component test report
+test_reports/

CHANGELOG.md

@@ -2,6 +2,38 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.54.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.54.0...v1.54.1) (2024-01-24)
+
+
+### Bug Fixes
+
+* remove brand mapping from products to contents in tiktok_ads ([49c4988](https://github.com/rudderlabs/rudder-transformer/commit/49c4988cc872f40780fa640240c209c3d768fab4))
+
+## [1.54.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.53.0...v1.54.0) (2024-01-22)
+
+
+### Features
+
+* add regulation to features.json ([2ae4378](https://github.com/rudderlabs/rudder-transformer/commit/2ae437820952b87dace289ec04d5c116d77cf70c))
+* add regulation to features.json ([#2988](https://github.com/rudderlabs/rudder-transformer/issues/2988)) ([65c79a8](https://github.com/rudderlabs/rudder-transformer/commit/65c79a8fd02218cee5d12e51004cdc023a0e7989))
+* onboard new tiktok version events 2.0 ([#2961](https://github.com/rudderlabs/rudder-transformer/issues/2961)) ([0fca088](https://github.com/rudderlabs/rudder-transformer/commit/0fca088980368954e9a82addedd6a4544035259a))
+
+
+### Bug Fixes
+
+* add valid code coverage file in workflows ([#3000](https://github.com/rudderlabs/rudder-transformer/issues/3000)) ([34bf49d](https://github.com/rudderlabs/rudder-transformer/commit/34bf49d19f33326008a467230574e4f1c78e9ddb))
+* error handling for trade desk first party data flow ([#2986](https://github.com/rudderlabs/rudder-transformer/issues/2986)) ([32c9c03](https://github.com/rudderlabs/rudder-transformer/commit/32c9c038ca96477cbe8d94d2a19679817b952e4c))
+* fixing network and default error messages for marketo bulk upload ([#3016](https://github.com/rudderlabs/rudder-transformer/issues/3016)) ([3741073](https://github.com/rudderlabs/rudder-transformer/commit/3741073ea37e83aa1e6e35c3ccb26d5a41a49617))
+* klaviyo validate event name to be string before accessing it ([#2998](https://github.com/rudderlabs/rudder-transformer/issues/2998)) ([7aa6687](https://github.com/rudderlabs/rudder-transformer/commit/7aa6687829783df3924a89698a53b9cb315449eb))
+* mailjet source operating on array instead object ([#2999](https://github.com/rudderlabs/rudder-transformer/issues/2999)) ([0dbb307](https://github.com/rudderlabs/rudder-transformer/commit/0dbb307bf5d18893e3331cd92bc16bae1b4a6add))
+* mixpanel alias simplified merge error message ([#2996](https://github.com/rudderlabs/rudder-transformer/issues/2996)) ([e53b304](https://github.com/rudderlabs/rudder-transformer/commit/e53b3046252bb5abcae046445baf4a67c40f37a3))
+* optimise build and coverage ([#2985](https://github.com/rudderlabs/rudder-transformer/issues/2985)) ([81ab6e8](https://github.com/rudderlabs/rudder-transformer/commit/81ab6e86070a9992141bad405c6f52d1ab186f11))
+* ortto remove phn:phone field if no phone is there in payload ([#2997](https://github.com/rudderlabs/rudder-transformer/issues/2997)) ([0c2f7bc](https://github.com/rudderlabs/rudder-transformer/commit/0c2f7bcbf705787087a8d9073145e282851ddb6f))
+* pr conflicts in back merge ([#2984](https://github.com/rudderlabs/rudder-transformer/issues/2984)) ([f16066c](https://github.com/rudderlabs/rudder-transformer/commit/f16066cc8f867378f1b9bb9812f8efe389ca5bc8))
+* **slack:** add support of blank space in handlebar ([#2959](https://github.com/rudderlabs/rudder-transformer/issues/2959)) ([ff12d8f](https://github.com/rudderlabs/rudder-transformer/commit/ff12d8ffddccbdac934c7cbac03730cd2593147f))
+* tiktok add missing field brand ([#2942](https://github.com/rudderlabs/rudder-transformer/issues/2942)) ([ca76297](https://github.com/rudderlabs/rudder-transformer/commit/ca76297147f0a2c12d68d97dda95b9a44efe4020))
+* tiktok_ads remove page.url as a required field ([#3006](https://github.com/rudderlabs/rudder-transformer/issues/3006)) ([595f819](https://github.com/rudderlabs/rudder-transformer/commit/595f81982a2604ec08621e15cea7c16564d5ecab))
+
 ## [1.53.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.52.4...v1.53.0) (2024-01-08)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.53.0",
+  "version": "1.54.1",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -110,7 +110,7 @@
     "stacktrace-parser": "^0.1.10",
     "statsd-client": "^0.4.7",
     "truncate-utf8-bytes": "^1.0.2",
-    "ua-parser-js": "^1.0.35",
+    "ua-parser-js": "^1.0.37",
     "unset-value": "^2.0.1",
     "uuid": "^9.0.0",
     "valid-url": "^1.0.9"

src/cdk/v2/destinations/ortto/procWorkflow.yaml

@@ -34,6 +34,7 @@ steps:
     description: |
       Builds common fields in destination payload.
     template: |
+      let phone = .message.().({{{{$.getGenericPaths("phone")}}}});
       let commonFields = .message.().({
         "fields": { 
           "str::first": {{{{$.getGenericPaths("firstName")}}}}, 
@@ -46,7 +47,7 @@ steps:
           "dtz::b": $.getBirthdayObj({{{{$.getGenericPaths("birthday")}}}}),
           "str::ei": {{{{$.getGenericPaths("userId")}}}},
           "str::language": .context.traits.language || .context.locale,
-          "phn::phone": {"n": {{{{$.getGenericPaths("phone")}}}}},
+          "phn::phone": phone ? {"n": phone},
           "bol::gdpr": .context.traits.gdpr ?? true,
           "bol::p": .context.traits.emailConsent || false,
           "bol::sp": .context.traits.smsConsent || false,

src/cdk/v2/destinations/the_trade_desk/utils.js

@@ -8,6 +8,7 @@ const {
   getSuccessRespEvents,
   removeUndefinedAndNullValues,
   handleRtTfSingleEventError,
+  isEmptyObject,
 } = require('../../../../v0/util');
 const tradeDeskConfig = require('./config');
 
@@ -59,34 +60,44 @@ const processRecordInputs = (inputs, destination) => {
   const successMetadata = [];
   const errorResponseList = [];
 
-  const error = new InstrumentationError('Invalid action type');
+  const invalidActionTypeError = new InstrumentationError('Invalid action type');
+  const emptyFieldsError = new InstrumentationError('Fields cannot be empty');
 
   inputs.forEach((input) => {
     const { fields, action } = input.message;
     const isInsertOrDelete = action === 'insert' || action === 'delete';
 
-    if (isInsertOrDelete) {
-      successMetadata.push(input.metadata);
-      const data = [
-        {
-          Name: Config.audienceId,
-          TTLInMinutes: action === 'insert' ? ttlInMin(Config.ttlInDays) : 0,
-        },
-      ];
-
-      Object.keys(fields).forEach((id) => {
-        const value = fields[id];
-        if (value) {
-          // adding only non empty ID's
-          items.push({ [id]: value, Data: data });
-        }
-      });
-    } else {
-      errorResponseList.push(handleRtTfSingleEventError(input, error, {}));
+    if (!isInsertOrDelete) {
+      errorResponseList.push(handleRtTfSingleEventError(input, invalidActionTypeError, {}));
+      return;
     }
+
+    if (isEmptyObject(fields)) {
+      errorResponseList.push(handleRtTfSingleEventError(input, emptyFieldsError, {}));
+      return;
+    }
+
+    successMetadata.push(input.metadata);
+    const data = [
+      {
+        Name: Config.audienceId,
+        TTLInMinutes: action === 'insert' ? ttlInMin(Config.ttlInDays) : 0,
+      },
+    ];
+
+    Object.keys(fields).forEach((id) => {
+      const value = fields[id];
+      if (value) {
+        // adding only non empty ID's
+        items.push({ [id]: value, Data: data });
+      }
+    });
   });
 
   const payloads = batchResponseBuilder(items, Config);
+  if (payloads.length === 0) {
+    return errorResponseList;
+  }
 
   const response = getSuccessRespEvents(payloads, successMetadata, destination, true);
   return [response, ...errorResponseList];

src/v0/destinations/klaviyo/transform.js

@@ -156,6 +156,9 @@ const trackRequestHandler = (message, category, destination) => {
   const payload = {};
   const { privateApiKey, flattenProperties } = destination.Config;
   let event = get(message, 'event');
+  if (event && typeof event !== 'string') {
+    throw new InstrumentationError('Event type should be a string');
+  }
   event = event ? event.trim().toLowerCase() : event;
   let attributes = {};
   if (ecomEvents.includes(event) && message.properties) {

src/v0/destinations/marketo_bulk_upload/fileUpload.js

@@ -206,8 +206,8 @@ const getImportID = async (input, config, accessToken, csvHeader) => {
     stats.counter('marketo_bulk_upload_upload_file_unsuccJobs', unsuccessfulJobs.length);
     if (!isHttpStatusSuccess(resp.status)) {
       throw new NetworkError(
-        'Unable to upload file',
-        hydrateStatusForServer(resp.status, 'During fetching poll status'),
+        `Unable to upload file due to error : ${resp.response}`,
+        hydrateStatusForServer(resp.status, 'During uploading file'),
       );
     }
     return handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime, config);

src/v0/destinations/marketo_bulk_upload/poll.js

@@ -34,7 +34,7 @@ const getPollStatus = async (event) => {
       state: 'Retryable',
     });
     throw new NetworkError(
-      'Could not poll status',
+      `Could not poll status: due to error ${pollStatus.response}`,
       hydrateStatusForServer(pollStatus.status, 'During fetching poll status'),
     );
   }

src/v0/destinations/marketo_bulk_upload/util.js

@@ -153,7 +153,7 @@ const getAccessToken = async (config) =>
     // sample response : {response: '[ENOTFOUND] :: DNS lookup failed', status: 400}
     if (!isHttpStatusSuccess(accessTokenResponse.status)) {
       throw new NetworkError(
-        'Could not retrieve authorisation token',
+        `Could not retrieve authorisation token due to error ${accessTokenResponse}`,
         hydrateStatusForServer(accessTokenResponse.status, FETCH_ACCESS_TOKEN),
         {
           [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(accessTokenResponse.status),
@@ -172,7 +172,7 @@ const getAccessToken = async (config) =>
 
     // when access token is present
     if (accessTokenResponse.response.access_token) {
-      /* This scenario will handle the case when we get the foloowing response
+      /* This scenario will handle the case when we get the following response
       status: 200  
       respnse: {"access_token":"<dummy-access-token>","token_type":"bearer","expires_in":0,"scope":"[email protected]"}
       wherein "expires_in":0 denotes that we should refresh the accessToken but its not expired yet. 
@@ -185,7 +185,10 @@ const getAccessToken = async (config) =>
       }
       return accessTokenResponse.response.access_token;
     }
-    throw new AbortedError('Could not retrieve authorisation token', 400);
+    throw new AbortedError(
+      `Could not retrieve authorisation token due to error ${accessTokenResponse}`,
+      400,
+    );
   });
 
 /**
@@ -254,14 +257,18 @@ const handleFetchJobStatusResponse = (resp, type) => {
   if (!isHttpStatusSuccess(marketoReposnseStatus)) {
     logger.info('[Network Error]:Failed during fetching job status', { marketoResponse, type });
     throw new NetworkError(
-      'Unable to fetch job status',
+      `Unable to fetch job status: due to error ${marketoResponse}`,
       hydrateStatusForServer(marketoReposnseStatus, 'During fetching job status'),
     );
   }
 
   if (marketoResponse?.success === false) {
     logger.info('[Application Error]Failed during fetching job status', { marketoResponse, type });
-    throw new RetryableError('Failure during fetching job status', 500, resp);
+    throw new RetryableError(
+      `Failure during fetching job status due to error : ${marketoResponse}`,
+      500,
+      resp,
+    );
   }
 
   /*
@@ -394,7 +401,11 @@ const getFieldSchemaMap = async (accessToken, munchkinId) => {
       fieldMap[field?.name] = field?.dataType;
     });
   } else {
-    throw new RetryableError('Failed to fetch Marketo Field Schema', 500, fieldSchemaMapping);
+    throw new RetryableError(
+      `Failed to fetch Marketo Field Schema due to error ${fieldSchemaMapping}`,
+      500,
+      fieldSchemaMapping,
+    );
   }
   return fieldMap;
 };

src/v0/destinations/mp/transform.js

@@ -441,7 +441,9 @@ const processSingleMessage = (message, destination) => {
       return processIdentifyEvents(clonedMessage, clonedMessage.type, destination);
     case EventType.ALIAS:
       if (destination.Config?.identityMergeApi === 'simplified') {
-        throw new InstrumentationError('Alias call is deprecated in `Simplified ID merge`');
+        throw new InstrumentationError(
+          'The use of the alias call in the context of the `Simplified ID merge` feature is not supported anymore.',
+        );
       }
       return processAliasEvents(message, message.type, destination);
     case EventType.GROUP: