Merge branch 'develop' into feat-linkedInConv

.github/workflows/prepare-for-staging-deploy.yml

@@ -112,9 +112,9 @@ jobs:
             yq eval -i ".user-transformer.image.tag=\"$TAG_NAME\"" staging.yaml
             git add staging.yaml
 
-            cd ../../../../config-be-rudder-transformer
-            yq eval -i ".config-be-rudder-transformer.image.tag=\"$TAG_NAME\"" values.staging.yaml
-            yq eval -i ".config-be-user-transformer.image.tag=\"$TAG_NAME\"" values.staging.yaml
+            cd ../../../../config-be-rudder-transformer/environment/staging
+            yq eval -i ".config-be-rudder-transformer.image.tag=\"$TAG_NAME\"" base.yaml
+            yq eval -i ".config-be-user-transformer.image.tag=\"$TAG_NAME\"" base.yaml
             git add values.staging.yaml
 
             git commit -m "chore: upgrade staging env transformers to \"$TAG_NAME\""

src/cdk/v2/destinations/movable_ink/config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  MAX_REQUEST_SIZE_IN_BYTES: 13500,
+};

src/cdk/v2/destinations/movable_ink/procWorkflow.yaml

@@ -0,0 +1,72 @@
+bindings:
+  - name: EventType
+    path: ../../../../constants
+  - path: ../../bindings/jsontemplate
+  - name: defaultRequestConfig
+    path: ../../../../v0/util
+  - name: toUnixTimestampInMS
+    path: ../../../../v0/util
+  - name: base64Convertor
+    path: ../../../../v0/util
+  - path: ./utils
+
+steps:
+  - name: messageType
+    template: |
+      .message.type.toLowerCase();
+
+  - name: validateInput
+    template: |
+      let messageType = $.outputs.messageType;
+      $.assert(messageType, "message Type is not present. Aborting");
+      $.assert(messageType in {{$.EventType.([.IDENTIFY,.TRACK])}}, "message type " + messageType + " is not supported");
+      $.assertConfig(.destination.Config.endpoint, "Movable Ink Endpoint is not present. Aborting");
+      $.assertConfig(.destination.Config.accessKey, "Access key is not present . Aborting");
+      $.assertConfig(.destination.Config.accessSecret, "Access Secret is not present. Aborting");
+      $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting");
+
+      const userId = .message.().(
+          {{{{$.getGenericPaths("userIdOnly")}}}};
+      );
+      const email = .message.().(
+          {{{{$.getGenericPaths("email")}}}};
+      );
+
+      $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting");
+
+  - name: preparePayload
+    description: Prepare payload for identify and track. This payload schema needs to be configured in the Movable Ink dashboard. Movable Ink will discard any additional fields from the input payload.
+    template: |
+      const userId = .message.().(
+          {{{{$.getGenericPaths("userIdOnly")}}}};
+      );
+      const email = .message.().(
+          {{{{$.getGenericPaths("email")}}}};
+      ); 
+      const timestampInUnix = $.toUnixTimestampInMS(.message.().(
+            {{{{$.getGenericPaths("timestamp")}}}};
+      ));
+      $.context.payload = {
+        ...(.message),
+        userId: userId ?? email,
+        timestamp: timestampInUnix,
+        anonymousId: .message.anonymousId
+      }
+
+  - name: buildResponse
+    description: In batchMode we return payload directly
+    condition: $.batchMode
+    template: |
+      $.context.payload
+    else:
+      name: buildResponseForProcessTransformation
+      template: |
+        const response = $.defaultRequestConfig();
+        response.body.JSON = $.context.payload;
+        response.endpoint = .destination.Config.endpoint;
+        response.method = "POST";
+        response.headers = {
+          "Content-Type": "application/json",
+          "Authorization": "Basic " + $.base64Convertor(.destination.Config.accessKey + ":" + .destination.Config.accessSecret)
+        }
+        response;

src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml

@@ -0,0 +1,74 @@
+bindings:
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
+  - path: ./utils
+    exportAll: true
+  - name: base64Convertor
+    path: ../../../../v0/util
+  - name: BatchUtils
+    path: '@rudderstack/workflow-engine'
+  - path: ./config
+
+steps:
+  - name: validateInput
+    template: |
+      $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+  - name: transform
+    externalWorkflow:
+      path: ./procWorkflow.yaml
+      bindings:
+        - name: batchMode
+          value: true
+    loopOverInput: true
+
+  - name: successfulEvents
+    template: |
+      $.outputs.transform#idx.output.({
+        "batchedRequest": .,
+        "batched": false,
+        "destination": ^[idx].destination,
+        "metadata": ^[idx].metadata,
+        "statusCode": 200
+      })[]
+
+  - name: failedEvents
+    template: |
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
+
+  - name: batchSuccessfulEvents
+    description: Batches the successfulEvents
+    template: |
+      let batches = $.BatchUtils.chunkArrayBySizeAndLength(
+      $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES}).items;
+
+      batches@batch.({
+        "batchedRequest": {
+          "body": {
+            "JSON": {"events": ~r batch.batchedRequest[]},
+            "JSON_ARRAY": {},
+            "XML": {},
+            "FORM": {}
+          },
+          "version": "1",
+          "type": "REST",
+          "method": "POST",
+          "endpoint": batch[0].destination.Config.().(.endpoint),
+          "headers": batch[0].destination.Config.().({
+            "Content-Type": "application/json",
+            "Authorization": "Basic " + $.base64Convertor(.accessKey + ":" + .accessSecret)
+          }),
+          "params": {},
+          "files": {}
+        },
+        "metadata": ~r batch.metadata[],
+        "batched": true,
+        "statusCode": 200,
+        "destination": batch[0].destination
+      })[];
+
+  - name: finalPayload
+    template: |
+      [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents]

src/features.json

@@ -68,7 +68,8 @@
     "INTERCOM": true,
     "NINETAILED": true,
     "KOALA": true,
-    "LINKEDIN_ADS": true
+    "LINKEDIN_ADS": true,
+    "MOVABLE_INK": true
   },
   "regulations": [
     "BRAZE",

src/v0/destinations/am/transform.js

@@ -525,6 +525,9 @@ const responseBuilderSimple = (
     ...campaign,
   };
 
+  // we are updating the payload with skip_user_properties_sync
+  AMUtils.updateWithSkipAttribute(message, rawPayload);
+
   const respData = getResponseData(evType, destination, rawPayload, message, groupInfo);
   const { groups, rawPayload: updatedRawPayload } = respData;
 

src/v0/destinations/am/util.test.js

@@ -1,4 +1,9 @@
-const { getUnsetObj, validateEventType, userPropertiesPostProcess } = require('./utils');
+const {
+  getUnsetObj,
+  validateEventType,
+  userPropertiesPostProcess,
+  updateWithSkipAttribute,
+} = require('./utils');
 
 describe('getUnsetObj', () => {
   it("should return undefined when 'message.integrations.Amplitude.fieldsToUnset' is not array", () => {
@@ -164,3 +169,31 @@ describe('userPropertiesPostProcess', () => {
     });
   });
 });
+
+describe('updateWithSkipAttribute', () => {
+  // when 'skipUserPropertiesSync ' is present in 'integrations.Amplitude', return the original payload.
+  it("should return the original payload when 'skipUserPropertiesSync' is present", () => {
+    const message = { integrations: { Amplitude: { skipUserPropertiesSync: true } } };
+    const payload = { key: 'value' };
+    const expectedPayload = { key: 'value', $skip_user_properties_sync: true };
+    updateWithSkipAttribute(message, payload);
+    expect(expectedPayload).toEqual(payload);
+  });
+
+  // When 'skipUserPropertiesSync' is not present in 'integrations.Amplitude', return the original payload.
+  it("should return the original payload when 'skipUserPropertiesSync' is not present", () => {
+    const message = { integrations: { Amplitude: {} } };
+    const payload = { key: 'value' };
+    const expectedPayload = { key: 'value' };
+    updateWithSkipAttribute(message, payload);
+    expect(payload).toEqual(expectedPayload);
+  });
+  // When 'message' is null, return null.
+  it("should return null when 'message' is null", () => {
+    const message = null;
+    const payload = { key: 'value' };
+    const expectedPayload = { key: 'value' };
+    updateWithSkipAttribute(message, payload);
+    expect(payload).toEqual(expectedPayload);
+  });
+});

src/v0/destinations/am/utils.js

@@ -11,6 +11,7 @@
 const get = require('get-value');
 const uaParser = require('@amplitude/ua-parser-js');
 const { InstrumentationError } = require('@rudderstack/integrations-lib');
+const set = require('set-value');
 const logger = require('../../../logger');
 const { isDefinedAndNotNull } = require('../../util');
 
@@ -110,6 +111,13 @@ const getUnsetObj = (message) => {
   return unsetObject;
 };
 
+const updateWithSkipAttribute = (message, payload) => {
+  const skipAttribute = get(message, 'integrations.Amplitude.skipUserPropertiesSync');
+  if (skipAttribute) {
+    set(payload, '$skip_user_properties_sync', true);
+  }
+};
+
 /**
  * Check for evType as in some cases, like when the page name is absent,
  * either the template depends only on the event.name or there is no template provided by user
@@ -187,4 +195,5 @@ module.exports = {
   getUnsetObj,
   validateEventType,
   userPropertiesPostProcess,
+  updateWithSkipAttribute,
 };