Merge branch 'develop' of github.com:rudderlabs/rudder-transformer in…

…to chore.dependabot-update

.github/workflows/prepare-for-staging-deploy.yml

@@ -112,9 +112,9 @@ jobs:
             yq eval -i ".user-transformer.image.tag=\"$TAG_NAME\"" staging.yaml
             git add staging.yaml
 
-            cd ../../../../config-be-rudder-transformer
-            yq eval -i ".config-be-rudder-transformer.image.tag=\"$TAG_NAME\"" values.staging.yaml
-            yq eval -i ".config-be-user-transformer.image.tag=\"$TAG_NAME\"" values.staging.yaml
+            cd ../../../../config-be-rudder-transformer/environment/staging
+            yq eval -i ".config-be-rudder-transformer.image.tag=\"$TAG_NAME\"" base.yaml
+            yq eval -i ".config-be-user-transformer.image.tag=\"$TAG_NAME\"" base.yaml
             git add values.staging.yaml
 
             git commit -m "chore: upgrade staging env transformers to \"$TAG_NAME\""

CHANGELOG.md

@@ -2,6 +2,28 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.59.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.59.0) (2024-03-18)
+
+
+### Features
+
+* add Koala destination ([#3122](https://github.com/rudderlabs/rudder-transformer/issues/3122)) ([1ca039d](https://github.com/rudderlabs/rudder-transformer/commit/1ca039d64ebb1a18a0fc6b78ed5ee08528ad6b48))
+* add support of skip_user_properties_sync on Amplitude ([#3181](https://github.com/rudderlabs/rudder-transformer/issues/3181)) ([5e4ddbd](https://github.com/rudderlabs/rudder-transformer/commit/5e4ddbd8a591341a581a5721505d6dcb010f2eec))
+* adding zod validations ([#3066](https://github.com/rudderlabs/rudder-transformer/issues/3066)) ([325433b](https://github.com/rudderlabs/rudder-transformer/commit/325433b9188c8d1dbe740c7e193cdc2e58fdd751))
+* onboard destination movable ink ([#3167](https://github.com/rudderlabs/rudder-transformer/issues/3167)) ([7018b1e](https://github.com/rudderlabs/rudder-transformer/commit/7018b1e5e7f37ae177191c5ecf3a71cfe2f3d147))
+* update proxy tests for cm360 ([#3039](https://github.com/rudderlabs/rudder-transformer/issues/3039)) ([0504ffa](https://github.com/rudderlabs/rudder-transformer/commit/0504ffa898956f5b61771fb32ecfd0e0bf15248f))
+* use dontBatch directive in algolia ([#3169](https://github.com/rudderlabs/rudder-transformer/issues/3169)) ([916aaec](https://github.com/rudderlabs/rudder-transformer/commit/916aaecb1939160620d5fd3c4c0c0e33f2a371b2))
+
+
+### Bug Fixes
+
+* api contract for v1 proxy ([#3049](https://github.com/rudderlabs/rudder-transformer/issues/3049)) ([93947db](https://github.com/rudderlabs/rudder-transformer/commit/93947db35cdaf1ca7ed87ec5f73567754af312ab))
+* email mapping for clevertap ([#3173](https://github.com/rudderlabs/rudder-transformer/issues/3173)) ([04eab92](https://github.com/rudderlabs/rudder-transformer/commit/04eab92e1c383f9e8cdd5c845530a42a0af2932a))
+* fb pixel test case refactor ([#3075](https://github.com/rudderlabs/rudder-transformer/issues/3075)) ([cff7d1c](https://github.com/rudderlabs/rudder-transformer/commit/cff7d1c4578087a37614c0ef4529058481873479))
+* fixed 500 status for algolia dontBatch ([#3178](https://github.com/rudderlabs/rudder-transformer/issues/3178)) ([6330888](https://github.com/rudderlabs/rudder-transformer/commit/6330888ad5c67e3a800037b56501fc08da09e4d1))
+* label not present in prometheus metrics ([#3176](https://github.com/rudderlabs/rudder-transformer/issues/3176)) ([01d460c](https://github.com/rudderlabs/rudder-transformer/commit/01d460c3edaf39b35c4686516c9e9140be46aa5e))
+* send proper status to server in cm360 ([#3127](https://github.com/rudderlabs/rudder-transformer/issues/3127)) ([229ce47](https://github.com/rudderlabs/rudder-transformer/commit/229ce473af1ddd62d946bea1b018c882b142a5ef))
+
 ## [1.58.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.58.0) (2024-03-04)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.58.0",
+  "version": "1.59.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/cdk/v2/destinations/movable_ink/config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  MAX_REQUEST_SIZE_IN_BYTES: 13500,
+};

src/cdk/v2/destinations/movable_ink/procWorkflow.yaml

@@ -0,0 +1,73 @@
+bindings:
+  - name: EventType
+    path: ../../../../constants
+  - path: ../../bindings/jsontemplate
+  - name: defaultRequestConfig
+    path: ../../../../v0/util
+  - name: toUnixTimestampInMS
+    path: ../../../../v0/util
+  - name: base64Convertor
+    path: ../../../../v0/util
+  - path: ./utils
+
+steps:
+  - name: messageType
+    template: |
+      .message.type.toLowerCase();
+
+  - name: validateInput
+    template: |
+      let messageType = $.outputs.messageType;
+      $.assert(messageType, "message Type is not present. Aborting");
+      $.assert(messageType in {{$.EventType.([.IDENTIFY,.TRACK])}}, "message type " + messageType + " is not supported");
+      $.assertConfig(.destination.Config.endpoint, "Movable Ink Endpoint is not present. Aborting");
+      $.assertConfig(.destination.Config.accessKey, "Access key is not present . Aborting");
+      $.assertConfig(.destination.Config.accessSecret, "Access Secret is not present. Aborting");
+      $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting");
+
+      const userId = .message.().(
+          {{{{$.getGenericPaths("userIdOnly")}}}};
+      );
+      const email = .message.().(
+          {{{{$.getGenericPaths("email")}}}};
+      );
+
+      $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting");
+      $.validateEventPayload(.message);
+
+  - name: preparePayload
+    description: Prepare payload for identify and track. This payload schema needs to be configured in the Movable Ink dashboard. Movable Ink will discard any additional fields from the input payload.
+    template: |
+      const userId = .message.().(
+          {{{{$.getGenericPaths("userIdOnly")}}}};
+      );
+      const email = .message.().(
+          {{{{$.getGenericPaths("email")}}}};
+      ); 
+      const timestampInUnix = $.toUnixTimestampInMS(.message.().(
+            {{{{$.getGenericPaths("timestamp")}}}};
+      ));
+      $.context.payload = {
+        ...(.message),
+        userId: userId ?? email,
+        timestamp: timestampInUnix,
+        anonymousId: .message.anonymousId
+      }
+
+  - name: buildResponse
+    description: In batchMode we return payload directly
+    condition: $.batchMode
+    template: |
+      $.context.payload
+    else:
+      name: buildResponseForProcessTransformation
+      template: |
+        const response = $.defaultRequestConfig();
+        response.body.JSON = $.context.payload;
+        response.endpoint = .destination.Config.endpoint;
+        response.method = "POST";
+        response.headers = {
+          "Content-Type": "application/json",
+          "Authorization": "Basic " + $.base64Convertor(.destination.Config.accessKey + ":" + .destination.Config.accessSecret)
+        }
+        response;

src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml

@@ -0,0 +1,74 @@
+bindings:
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
+  - path: ./utils
+    exportAll: true
+  - name: base64Convertor
+    path: ../../../../v0/util
+  - name: BatchUtils
+    path: '@rudderstack/workflow-engine'
+  - path: ./config
+
+steps:
+  - name: validateInput
+    template: |
+      $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+  - name: transform
+    externalWorkflow:
+      path: ./procWorkflow.yaml
+      bindings:
+        - name: batchMode
+          value: true
+    loopOverInput: true
+
+  - name: successfulEvents
+    template: |
+      $.outputs.transform#idx.output.({
+        "batchedRequest": .,
+        "batched": false,
+        "destination": ^[idx].destination,
+        "metadata": ^[idx].metadata,
+        "statusCode": 200
+      })[]
+
+  - name: failedEvents
+    template: |
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
+
+  - name: batchSuccessfulEvents
+    description: Batches the successfulEvents
+    template: |
+      let batches = $.BatchUtils.chunkArrayBySizeAndLength(
+      $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES}).items;
+
+      batches@batch.({
+        "batchedRequest": {
+          "body": {
+            "JSON": {"events": ~r batch.batchedRequest[]},
+            "JSON_ARRAY": {},
+            "XML": {},
+            "FORM": {}
+          },
+          "version": "1",
+          "type": "REST",
+          "method": "POST",
+          "endpoint": batch[0].destination.Config.().(.endpoint),
+          "headers": batch[0].destination.Config.().({
+            "Content-Type": "application/json",
+            "Authorization": "Basic " + $.base64Convertor(.accessKey + ":" + .accessSecret)
+          }),
+          "params": {},
+          "files": {}
+        },
+        "metadata": ~r batch.metadata[],
+        "batched": true,
+        "statusCode": 200,
+        "destination": batch[0].destination
+      })[];
+
+  - name: finalPayload
+    template: |
+      [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents]

src/cdk/v2/destinations/movable_ink/utils.js

@@ -0,0 +1,21 @@
+const { InstrumentationError } = require('@rudderstack/integrations-lib');
+
+const validateEventPayload = (message) => {
+  const { event } = message;
+  const { properties } = message;
+  if (event === 'Products Searched' && !properties?.query) {
+    throw new InstrumentationError("Missing 'query' property in properties. Aborting");
+  }
+
+  if (
+    (event === 'Product Added' ||
+      event === 'Product Removed' ||
+      event === 'Product Viewed' ||
+      event === 'Category Viewed') &&
+    !properties?.product_id
+  ) {
+    throw new InstrumentationError("Missing 'product_id' property in properties. Aborting");
+  }
+};
+
+module.exports = { validateEventPayload };

src/features.json

@@ -67,6 +67,7 @@
     "THE_TRADE_DESK": true,
     "INTERCOM": true,
     "NINETAILED": true,
+    "MOVABLE_INK": true,
     "KOALA": true
   },
   "regulations": [

src/v0/destinations/am/transform.js

@@ -525,6 +525,9 @@ const responseBuilderSimple = (
     ...campaign,
   };
 
+  // we are updating the payload with skip_user_properties_sync
+  AMUtils.updateWithSkipAttribute(message, rawPayload);
+
   const respData = getResponseData(evType, destination, rawPayload, message, groupInfo);
   const { groups, rawPayload: updatedRawPayload } = respData;
 

src/v0/destinations/am/util.test.js

@@ -1,4 +1,9 @@
-const { getUnsetObj, validateEventType, userPropertiesPostProcess } = require('./utils');
+const {
+  getUnsetObj,
+  validateEventType,
+  userPropertiesPostProcess,
+  updateWithSkipAttribute,
+} = require('./utils');
 
 describe('getUnsetObj', () => {
   it("should return undefined when 'message.integrations.Amplitude.fieldsToUnset' is not array", () => {
@@ -164,3 +169,31 @@ describe('userPropertiesPostProcess', () => {
     });
   });
 });
+
+describe('updateWithSkipAttribute', () => {
+  // when 'skipUserPropertiesSync ' is present in 'integrations.Amplitude', return the original payload.
+  it("should return the original payload when 'skipUserPropertiesSync' is present", () => {
+    const message = { integrations: { Amplitude: { skipUserPropertiesSync: true } } };
+    const payload = { key: 'value' };
+    const expectedPayload = { key: 'value', $skip_user_properties_sync: true };
+    updateWithSkipAttribute(message, payload);
+    expect(expectedPayload).toEqual(payload);
+  });
+
+  // When 'skipUserPropertiesSync' is not present in 'integrations.Amplitude', return the original payload.
+  it("should return the original payload when 'skipUserPropertiesSync' is not present", () => {
+    const message = { integrations: { Amplitude: {} } };
+    const payload = { key: 'value' };
+    const expectedPayload = { key: 'value' };
+    updateWithSkipAttribute(message, payload);
+    expect(payload).toEqual(expectedPayload);
+  });
+  // When 'message' is null, return null.
+  it("should return null when 'message' is null", () => {
+    const message = null;
+    const payload = { key: 'value' };
+    const expectedPayload = { key: 'value' };
+    updateWithSkipAttribute(message, payload);
+    expect(payload).toEqual(expectedPayload);
+  });
+});

src/v0/destinations/am/utils.js

@@ -11,6 +11,7 @@
 const get = require('get-value');
 const uaParser = require('@amplitude/ua-parser-js');
 const { InstrumentationError } = require('@rudderstack/integrations-lib');
+const set = require('set-value');
 const logger = require('../../../logger');
 const { isDefinedAndNotNull } = require('../../util');
 
@@ -110,6 +111,13 @@ const getUnsetObj = (message) => {
   return unsetObject;
 };
 
+const updateWithSkipAttribute = (message, payload) => {
+  const skipAttribute = get(message, 'integrations.Amplitude.skipUserPropertiesSync');
+  if (skipAttribute) {
+    set(payload, '$skip_user_properties_sync', true);
+  }
+};
+
 /**
  * Check for evType as in some cases, like when the page name is absent,
  * either the template depends only on the event.name or there is no template provided by user
@@ -187,4 +195,5 @@ module.exports = {
   getUnsetObj,
   validateEventType,
   userPropertiesPostProcess,
+  updateWithSkipAttribute,
 };