Merge branch 'develop' of github.com:rudderlabs/rudder-transformer in…

…to feat.ga4-component-test-refactor
rudderlabs · Apr 25, 2024 · 281f4ca · 281f4ca
2 parents 159ac95 + 89a71b7
commit 281f4ca
Show file tree

Hide file tree

Showing 134 changed files with 7,747 additions and 8,462 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -26,6 +26,7 @@
       "off",
       { "cases": { "camelCase": true, "pascalCase": true, "kebabCase": true } }
     ],
+    "import/no-import-module-exports": "off",
     "unicorn/no-instanceof-array": "error",
     "unicorn/no-static-only-class": "error",
     "unicorn/consistent-destructuring": "error",

diff --git a/.github/workflows/publish-new-release.yml b/.github/workflows/publish-new-release.yml
@@ -99,7 +99,7 @@ jobs:
           channel-id: ${{ secrets.SLACK_RELEASE_CHANNEL_ID }}
           payload: |
             {
-              "text": "*<${{env.RELEASES_URL}}v${{ steps.extract-version.outputs.release_version }}|v${{ steps.extract-version.outputs.release_version }}>*\nCC: <@U03KG4BK1L1> <@U02AE5GMMHV> <@U01LVJ30QEB>",
+              "text": "*<${{env.RELEASES_URL}}v${{ steps.extract-version.outputs.release_version }}|v${{ steps.extract-version.outputs.release_version }}>*\nCC: <@U03KG4BK1L1> <@U024YF8CR53> <@U01LVJ30QEB>",
               "blocks": [
                 {
                   "type": "header",
@@ -115,7 +115,7 @@ jobs:
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "*<${{env.RELEASES_URL}}v${{ steps.extract-version.outputs.release_version }}|v${{ steps.extract-version.outputs.release_version }}>*\nCC: <@U03KG4BK1L1> <@U02AE5GMMHV> <@U01LVJ30QEB>"
+                    "text": "*<${{env.RELEASES_URL}}v${{ steps.extract-version.outputs.release_version }}|v${{ steps.extract-version.outputs.release_version }}>*\nCC: <@U03KG4BK1L1> <@U024YF8CR53> <@U01LVJ30QEB>"
                   }
                 }
               ]

diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-18.19.1
+18.20.1
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,64 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.62.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.62.1...v1.62.2) (2024-04-18)
+
+
+### Bug Fixes
+
+* twitter_ads logger ([#3295](https://github.com/rudderlabs/rudder-transformer/issues/3295)) ([e92b052](https://github.com/rudderlabs/rudder-transformer/commit/e92b052e03182deb41b20b3ec3741306afa50380))
+
+### [1.62.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.62.0...v1.62.1) (2024-04-18)
+
+
+### Bug Fixes
+
+* revert mixpanel deprecate /track ([#3291](https://github.com/rudderlabs/rudder-transformer/issues/3291)) ([ec068b4](https://github.com/rudderlabs/rudder-transformer/commit/ec068b49bd4a5652a762c60a8257c883e4709d1a))
+
+## [1.62.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.61.1...v1.62.0) (2024-04-15)
+
+
+### Features
+
+* do away myaxios ([#3222](https://github.com/rudderlabs/rudder-transformer/issues/3222)) ([9214594](https://github.com/rudderlabs/rudder-transformer/commit/9214594bab2c86a4ae6f75e12531f778490cf127))
+* for reddit adding currency and value for addToCart, viewConent event as well ([#3239](https://github.com/rudderlabs/rudder-transformer/issues/3239)) ([ad235e7](https://github.com/rudderlabs/rudder-transformer/commit/ad235e785bf6039e11231a915be098130b25ec3b))
+* logger upgrade in services, dest, source ([#3228](https://github.com/rudderlabs/rudder-transformer/issues/3228)) ([c204113](https://github.com/rudderlabs/rudder-transformer/commit/c204113eab37a782f217488d0d626a8d6df345d3))
+* rakuten: adding a default value for tr ([#3240](https://github.com/rudderlabs/rudder-transformer/issues/3240)) ([3748f24](https://github.com/rudderlabs/rudder-transformer/commit/3748f24e21634fc74c5e5b3761551c64c8e69942))
+
+
+### Bug Fixes
+
+* adding check for reserved key words in extract custom fields  ([#3264](https://github.com/rudderlabs/rudder-transformer/issues/3264)) ([3399c47](https://github.com/rudderlabs/rudder-transformer/commit/3399c47fdce1b3d19e29306ca3c5692a2fbc30fb))
+* deployment file paths ([#3216](https://github.com/rudderlabs/rudder-transformer/issues/3216)) ([808727d](https://github.com/rudderlabs/rudder-transformer/commit/808727de17e400ed102a843ab3b30f81f8900f24))
+* email mappings ([#3247](https://github.com/rudderlabs/rudder-transformer/issues/3247)) ([791cbf5](https://github.com/rudderlabs/rudder-transformer/commit/791cbf55fc6940af4e3208212b82c891c6618fc3))
+* fixed userId mapping, now mapping to uid instead of id ([#3262](https://github.com/rudderlabs/rudder-transformer/issues/3262)) ([9c6b251](https://github.com/rudderlabs/rudder-transformer/commit/9c6b251a6c784cc391f27e846a008fbe2901e2c8))
+* hs bugsnag error ([#3252](https://github.com/rudderlabs/rudder-transformer/issues/3252)) ([9daf1c9](https://github.com/rudderlabs/rudder-transformer/commit/9daf1c989258bd410d5780c1b11c4f6df9654af5))
+* hubspot: search for contact using secondary prop ([#3258](https://github.com/rudderlabs/rudder-transformer/issues/3258)) ([0b57204](https://github.com/rudderlabs/rudder-transformer/commit/0b5720446693efe1fd0ccdfc141bd7f21b2c32ae))
+* impact: support custom product mapping ([#3249](https://github.com/rudderlabs/rudder-transformer/issues/3249)) ([cb8ff2f](https://github.com/rudderlabs/rudder-transformer/commit/cb8ff2fb943c49df4ac083bd179d9674b40eb602))
+* marketo bulk ignore null while checking data type mismatch ([#3263](https://github.com/rudderlabs/rudder-transformer/issues/3263)) ([6e3274b](https://github.com/rudderlabs/rudder-transformer/commit/6e3274bfba9e7838d1f81d845a070427b67e75f5))
+* shopify: send 500 for identifier call in case of failure ([#3235](https://github.com/rudderlabs/rudder-transformer/issues/3235)) ([8eb4c4e](https://github.com/rudderlabs/rudder-transformer/commit/8eb4c4e9b8daebbaeb1d12ff0c17915fe19c2b50))
+
+### [1.61.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.61.0...v1.61.1) (2024-04-03)
+
+## [1.61.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.60.0...v1.61.0) (2024-04-02)
+
+
+### Features
+
+* consent field support for ga4 ([#3213](https://github.com/rudderlabs/rudder-transformer/issues/3213)) ([92515a5](https://github.com/rudderlabs/rudder-transformer/commit/92515a5fd8a2798c48010078f62b360ec6a49979))
+* consent field support for gaoc and upgrade the api version from v14 to v16 ([#3121](https://github.com/rudderlabs/rudder-transformer/issues/3121)) ([2aac2a6](https://github.com/rudderlabs/rudder-transformer/commit/2aac2a62547b7a7c617735fc3d6e88e0a1bed76e)), closes [#3190](https://github.com/rudderlabs/rudder-transformer/issues/3190)
+* onboard new destination bloomreach ([#3185](https://github.com/rudderlabs/rudder-transformer/issues/3185)) ([d9b7e1f](https://github.com/rudderlabs/rudder-transformer/commit/d9b7e1f70565d59979aee3e62f60e39edb9a23c7))
+* onboarding linkedin conversion api ([#3194](https://github.com/rudderlabs/rudder-transformer/issues/3194)) ([eb7b197](https://github.com/rudderlabs/rudder-transformer/commit/eb7b197322c617b14c2579de8cb4d4dacf8e1df3))
+* update movable ink batch size ([#3223](https://github.com/rudderlabs/rudder-transformer/issues/3223)) ([667095f](https://github.com/rudderlabs/rudder-transformer/commit/667095fa8316cd95a066f15b848ad503c6b4af80))
+
+
+### Bug Fixes
+
+* fixed userId mapping, now mapping to uid instead of id ([#3192](https://github.com/rudderlabs/rudder-transformer/issues/3192)) ([70a468b](https://github.com/rudderlabs/rudder-transformer/commit/70a468bf16ecd5ee0b6fecee4b837895d19c525f))
+* ninetailed: remove  page support ([#3218](https://github.com/rudderlabs/rudder-transformer/issues/3218)) ([2f30c56](https://github.com/rudderlabs/rudder-transformer/commit/2f30c56af62e983d09b5d4f2da9a0ba22f5c1612))
+* shopify invalid_event metric prometheus label ([#3200](https://github.com/rudderlabs/rudder-transformer/issues/3200)) ([345c87d](https://github.com/rudderlabs/rudder-transformer/commit/345c87d7c530c621ae3fd6c504d64e5a14e31f22))
+* fix: snapchat conversion: add event level_complete ([#3231](https://github.com/rudderlabs/rudder-transformer/issues/3231)) ([39368a0](https://github.com/rudderlabs/rudder-transformer/commit/39368a09e48acc324faa855186bc623e5c347881))
+
 ## [1.60.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.60.0) (2024-03-20)
 
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.4
-FROM node:18.19.1-alpine3.18 AS base
+FROM node:18.20.1-alpine3.18 AS base
 ENV HUSKY 0
 
 RUN apk update

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.60.0",
+  "version": "1.62.2",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -63,8 +63,8 @@
     "@datadog/pprof": "^3.1.0",
     "@koa/router": "^12.0.0",
     "@ndhoule/extend": "^2.0.0",
-    "@pyroscope/nodejs": "^0.2.6",
-    "@rudderstack/integrations-lib": "^0.2.7",
+    "@pyroscope/nodejs": "^0.2.9",
+    "@rudderstack/integrations-lib": "^0.2.8",
     "@rudderstack/workflow-engine": "^0.7.5",
     "@shopify/jest-koa-mocks": "^5.1.1",
     "ajv": "^8.12.0",

diff --git a/src/cdk/v2/destinations/bluecore/config.js b/src/cdk/v2/destinations/bluecore/config.js
@@ -1,6 +1,6 @@
 const { getMappingConfig } = require('../../../../v0/util');
 
-const BASE_URL = 'https://api.bluecore.com/api/track/mobile/v1';
+const BASE_URL = 'https://api.bluecore.app/api/track/mobile/v1';
 
 const CONFIG_CATEGORIES = {
   IDENTIFY: {
@@ -46,11 +46,31 @@ const EVENT_NAME_MAPPING = [
 
 const BLUECORE_EXCLUSION_FIELDS = ['query', 'order_id', 'total'];
 
+const IDENTIFY_EXCLUSION_LIST = [
+  'name',
+  'firstName',
+  'first_name',
+  'firstname',
+  'lastName',
+  'last_name',
+  'lastname',
+  'email',
+  'age',
+  'sex',
+  'address',
+  'action',
+  'event',
+];
+
+const TRACK_EXCLUSION_LIST = [...IDENTIFY_EXCLUSION_LIST, 'query', 'order_id', 'total', 'products'];
+
 const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname);
 module.exports = {
   CONFIG_CATEGORIES,
   MAPPING_CONFIG,
   EVENT_NAME_MAPPING,
   BASE_URL,
   BLUECORE_EXCLUSION_FIELDS,
+  IDENTIFY_EXCLUSION_LIST,
+  TRACK_EXCLUSION_LIST,
 };
diff --git a/src/cdk/v2/destinations/bluecore/data/bluecoreCommonConfig.json b/src/cdk/v2/destinations/bluecore/data/bluecoreCommonConfig.json
@@ -35,7 +35,7 @@
   },
   {
     "destKey": "properties.customer.email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": false,
     "sourceFromGenericMap": true
   },

diff --git a/src/cdk/v2/destinations/bluecore/procWorkflow.yaml b/src/cdk/v2/destinations/bluecore/procWorkflow.yaml
@@ -26,7 +26,7 @@ steps:
     condition: $.outputs.messageType === {{$.EventType.IDENTIFY}}
     template: |
       const payload = $.constructProperties(.message);
-      payload.token = .destination.Config.bluecoreNamespace;
+      payload.properties.token = .destination.Config.bluecoreNamespace;
       $.verifyPayload(payload, .message);
       payload.event = payload.event ?? 'customer_patch';
       payload.properties.distinct_id = $.populateAccurateDistinctId(payload, .message);
@@ -50,7 +50,7 @@ steps:
             const temporaryProductArray = newPayload.properties.products ?? $.createProductForStandardEcommEvent(^.message, eventName);
             newPayload.properties.products = $.normalizeProductArray(temporaryProductArray);
             newPayload.event = eventName;
-            newPayload.token = ^.destination.Config.bluecoreNamespace;
+            newPayload.properties.token = ^.destination.Config.bluecoreNamespace;
             $.verifyPayload(newPayload, ^.message);
             $.removeUndefinedNullValuesAndEmptyObjectArray(newPayload)
           )[];
@@ -61,7 +61,7 @@ steps:
         const response = $.defaultRequestConfig();
         response.body.JSON = .;
         response.method = "POST";
-        response.endpoint = "https://api.bluecore.com/api/track/mobile/v1";
+        response.endpoint = "https://api.bluecore.app/api/track/mobile/v1";
         response.headers = {
           "Content-Type": "application/json"
         };

diff --git a/src/cdk/v2/destinations/bluecore/utils.js b/src/cdk/v2/destinations/bluecore/utils.js
@@ -12,9 +12,10 @@ const {
   validateEventName,
   constructPayload,
   getDestinationExternalID,
+  extractCustomFields,
 } = require('../../../../v0/util');
 const { CommonUtils } = require('../../../../util/common');
-const { EVENT_NAME_MAPPING } = require('./config');
+const { EVENT_NAME_MAPPING, IDENTIFY_EXCLUSION_LIST, TRACK_EXCLUSION_LIST } = require('./config');
 const { EventType } = require('../../../../constants');
 const { MAPPING_CONFIG, CONFIG_CATEGORIES } = require('./config');
 
@@ -167,6 +168,41 @@ const normalizeProductArray = (products) => {
   return finalProductArray;
 };
 
+const mapCustomProperties = (message) => {
+  let customerProperties;
+  const customProperties = { properties: {} };
+  const messageType = message.type.toUpperCase();
+  switch (messageType) {
+    case 'IDENTIFY':
+      customerProperties = extractCustomFields(
+        message,
+        {},
+        ['traits', 'context.traits'],
+        IDENTIFY_EXCLUSION_LIST,
+      );
+      customProperties.properties.customer = customerProperties;
+      break;
+    case 'TRACK':
+      customerProperties = extractCustomFields(
+        message,
+        {},
+        ['traits', 'context.traits'],
+        IDENTIFY_EXCLUSION_LIST,
+      );
+      customProperties.properties = extractCustomFields(
+        message,
+        {},
+        ['properties'],
+        TRACK_EXCLUSION_LIST,
+      );
+      customProperties.properties.customer = customerProperties;
+      break;
+    default:
+      break;
+  }
+  return customProperties;
+};
+
 /**
  * Constructs properties based on the given message.
  *
@@ -178,7 +214,12 @@ const constructProperties = (message) => {
   const commonPayload = constructPayload(message, MAPPING_CONFIG[commonCategory.name]);
   const category = CONFIG_CATEGORIES[message.type.toUpperCase()];
   const typeSpecificPayload = constructPayload(message, MAPPING_CONFIG[category.name]);
-  const finalPayload = lodash.merge(commonPayload, typeSpecificPayload);
+  const typeSpecificCustomProperties = mapCustomProperties(message);
+  const finalPayload = lodash.merge(
+    commonPayload,
+    typeSpecificPayload,
+    typeSpecificCustomProperties,
+  );
   return finalPayload;
 };
 

diff --git a/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml b/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml
@@ -1,6 +1,8 @@
 bindings:
   - path: ./utils
   - path: ./config
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
 
 steps:
   - name: validateInput
@@ -23,14 +25,9 @@ steps:
       })[]
   - name: failedEvents
     template: |
-      $.outputs.transform#idx.error.({
-        "metadata": ^[idx].metadata[],
-        "destination": ^[idx].destination,
-        "batched": false,
-        "statusCode": .status,
-        "error": .message,
-        "statTags": .originalError.statTags
-      })[]
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
 
   - name: batchSuccessfulEvents
     description: Batches the successfulEvents

diff --git a/src/cdk/v2/destinations/linkedin_ads/utils.js b/src/cdk/v2/destinations/linkedin_ads/utils.js
@@ -162,6 +162,9 @@ const fetchAndVerifyConversionHappenedAt = (message) => {
 };
 
 function batchResponseBuilder(successfulEvents) {
+  if (successfulEvents.length === 0) {
+    return [];
+  }
   const constants = {
     version: successfulEvents[0].message[0].version,
     type: successfulEvents[0].message[0].type,