Merge branch 'feat-linkedInConv' of github.com:rudderlabs/rudder-tran…

…sformer into feat-linkedInConv

src/cdk/v2/destinations/bloomreach/config.ts

@@ -0,0 +1,30 @@
+import { getMappingConfig } from '../../../../v0/util';
+
+export const CUSTOMER_COMMAND = 'customers';
+export const CUSTOMER_EVENT_COMMAND = 'customers/events';
+export const MAX_BATCH_SIZE = 50;
+
+// ref:- https://documentation.bloomreach.com/engagement/reference/batch-commands-2
+export const getBatchEndpoint = (apiBaseUrl: string, projectToken: string): string =>
+  `${apiBaseUrl}/track/v2/projects/${projectToken}/batch`;
+
+const CONFIG_CATEGORIES = {
+  CUSTOMER_PROPERTIES_CONFIG: { name: 'BloomreachCustomerPropertiesConfig' },
+};
+const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname);
+export const EXCLUSION_FIELDS: string[] = [
+  'email',
+  'firstName',
+  'firstname',
+  'first_name',
+  'lastName',
+  'lastname',
+  'last_name',
+  'name',
+  'phone',
+  'city',
+  'birthday',
+  'country',
+];
+export const CUSTOMER_PROPERTIES_CONFIG =
+  MAPPING_CONFIG[CONFIG_CATEGORIES.CUSTOMER_PROPERTIES_CONFIG.name];

src/cdk/v2/destinations/bloomreach/data/BloomreachCustomerPropertiesConfig.json

@@ -0,0 +1,36 @@
+[
+  {
+    "destKey": "first_name",
+    "sourceKeys": "firstName",
+    "sourceFromGenericMap": true
+  },
+  {
+    "destKey": "last_name",
+    "sourceKeys": "lastName",
+    "sourceFromGenericMap": true
+  },
+  {
+    "destKey": "email",
+    "sourceKeys": "emailOnly",
+    "sourceFromGenericMap": true
+  },
+  {
+    "destKey": "phone",
+    "sourceKeys": "phone",
+    "sourceFromGenericMap": true
+  },
+  {
+    "destKey": "city",
+    "sourceKeys": "city",
+    "sourceFromGenericMap": true
+  },
+  {
+    "destKey": "country",
+    "sourceKeys": ["traits.address.country", "context.traits.address.country"]
+  },
+  {
+    "destKey": "birthday",
+    "sourceKeys": "birthday",
+    "sourceFromGenericMap": true
+  }
+]

src/cdk/v2/destinations/bloomreach/procWorkflow.yaml

@@ -0,0 +1,119 @@
+bindings:
+  - name: EventType
+    path: ../../../../constants
+  - path: ../../bindings/jsontemplate
+  - name: defaultRequestConfig
+    path: ../../../../v0/util
+  - name: toUnixTimestamp
+    path: ../../../../v0/util
+  - name: base64Convertor
+    path: ../../../../v0/util
+  - name: removeUndefinedAndNullValues
+    path: ../../../../v0/util
+  - name: generateExclusionList
+    path: ../../../../v0/util
+  - name: extractCustomFields
+    path: ../../../../v0/util
+  - name: constructPayload
+    path: ../../../../v0/util
+  - path: ./utils
+  - path: ./config
+
+steps:
+  - name: messageType
+    template: |
+      $.context.messageType = .message.type.toLowerCase();
+
+  - name: validateInput
+    template: |
+      let messageType = $.context.messageType;
+      $.assert(messageType, "message Type is not present. Aborting");
+      $.assert(messageType in {{$.EventType.([.IDENTIFY,.TRACK,.PAGE,.SCREEN])}}, "message type " + messageType + " is not supported");
+      $.assertConfig(.destination.Config.apiBaseUrl, "API Base URL is not present. Aborting");
+      $.assertConfig(.destination.Config.apiKey, "API Key is not present . Aborting");
+      $.assertConfig(.destination.Config.apiSecret, "API Secret is not present. Aborting");
+      $.assertConfig(.destination.Config.projectToken, "Project Token is not present. Aborting");
+      $.assertConfig(.destination.Config.hardID, "Hard ID is not present. Aborting");
+      $.assertConfig(.destination.Config.softID, "Soft ID is not present. Aborting");
+      $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting");
+      const userId = .message.().(
+          {{{{$.getGenericPaths("userIdOnly")}}}};
+      );
+      $.assert(userId ?? .message.anonymousId, "Either one of userId or anonymousId is required. Aborting");
+
+  - name: prepareIdentifyPayload
+    condition: $.context.messageType === {{$.EventType.IDENTIFY}}
+    template: |
+      const customerIDs = $.prepareCustomerIDs(.message, .destination);
+      const customerProperties = $.constructPayload(.message, $.CUSTOMER_PROPERTIES_CONFIG);
+      const extraCustomerProperties = $.extractCustomFields(.message, {}, ['traits', 'context.traits'], $.EXCLUSION_FIELDS);
+      const properties = {
+        ...customerProperties,
+        ...extraCustomerProperties
+      }
+      const data = .message.().({
+        "customer_ids": customerIDs,
+        "update_timestamp": $.toUnixTimestamp({{{{$.getGenericPaths("timestamp")}}}}),
+        properties
+      });
+
+      $.context.payload = $.removeUndefinedAndNullValues({name: $.CUSTOMER_COMMAND, data})
+
+  - name: prepareEventName
+    steps:
+      - name: pageEventName
+        condition: $.context.messageType === {{$.EventType.PAGE}}
+        template: |
+          const category = .message.category ?? .message.properties.category;
+          const name = .message.name || .message.properties.name;
+          const eventNameArray = ["Viewed"];
+          category ? eventNameArray.push(category);
+          name ? eventNameArray.push(name);
+          eventNameArray.push("Page");
+          $.context.event = eventNameArray.join(" ");
+      - name: screenEventName
+        condition: $.context.messageType === {{$.EventType.SCREEN}}
+        template: |
+          const category = .message.category ?? .message.properties.category;
+          const name = .message.name || .message.properties.name;
+          const eventNameArray = ["Viewed"];
+          category ? eventNameArray.push(category);
+          name ? eventNameArray.push(name);
+          eventNameArray.push("Screen");
+          $.context.event = eventNameArray.join(" ");
+      - name: trackEventName
+        condition: $.context.messageType === {{$.EventType.TRACK}}
+        template: |
+          $.assert(.message.event, "Event name is required. Aborting");
+          $.context.event = .message.event
+
+  - name: prepareTrackPageScreenPayload
+    condition: $.context.messageType !== {{$.EventType.IDENTIFY}}
+    template: |
+      const customerIDs = $.prepareCustomerIDs(.message, .destination);
+      const data = .message.().({
+        "customer_ids": customerIDs,
+        "timestamp": $.toUnixTimestamp({{{{$.getGenericPaths("timestamp")}}}}),
+        "properties": .properties,
+        "event_type": $.context.event,
+      });
+
+      $.context.payload = $.removeUndefinedAndNullValues({name: $.CUSTOMER_EVENT_COMMAND, data})
+
+  - name: buildResponse
+    description: In batchMode we return payload directly
+    condition: $.batchMode
+    template: |
+      $.context.payload
+    else:
+      name: buildResponseForProcessTransformation
+      template: |
+        const response = $.defaultRequestConfig();
+        response.body.JSON = $.context.payload;
+        response.endpoint = $.getBatchEndpoint(.destination.Config.apiBaseUrl, .destination.Config.projectToken);
+        response.method = "POST";
+        response.headers = {
+          "Content-Type": "application/json",
+          "Authorization": "Basic " + $.base64Convertor(.destination.Config.apiKey + ":" + .destination.Config.apiSecret)
+        }
+        response;

src/cdk/v2/destinations/bloomreach/rtWorkflow.yaml

@@ -0,0 +1,76 @@
+bindings:
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
+  - path: ./utils
+    exportAll: true
+  - name: base64Convertor
+    path: ../../../../v0/util
+  - name: toUnixTimestamp
+    path: ../../../../v0/util
+  - name: BatchUtils
+    path: '@rudderstack/workflow-engine'
+  - path: ./config
+
+steps:
+  - name: validateInput
+    template: |
+      $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+  - name: transform
+    externalWorkflow:
+      path: ./procWorkflow.yaml
+      bindings:
+        - name: batchMode
+          value: true
+    loopOverInput: true
+
+  - name: successfulEvents
+    template: |
+      $.outputs.transform#idx.output.({
+        "batchedRequest": .,
+        "batched": false,
+        "destination": ^[idx].destination,
+        "metadata": ^[idx].metadata,
+        "statusCode": 200
+      })[]
+
+  - name: failedEvents
+    template: |
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
+
+  - name: batchSuccessfulEvents
+    description: Batches the successfulEvents
+    template: |
+      let batches = $.BatchUtils.chunkArrayBySizeAndLength(
+      $.outputs.successfulEvents, {maxItems: $.MAX_BATCH_SIZE}).items;
+
+      batches@batch.({
+        "batchedRequest": {
+          "body": {
+            "JSON": {"commands": ~r batch.batchedRequest[]},
+            "JSON_ARRAY": {},
+            "XML": {},
+            "FORM": {}
+          },
+          "version": "1",
+          "type": "REST",
+          "method": "POST",
+          "endpoint": batch[0].destination.Config.().($.getBatchEndpoint(.apiBaseUrl, .projectToken)),
+          "headers": batch[0].destination.Config.().({
+            "Content-Type": "application/json",
+            "Authorization": "Basic " + $.base64Convertor(.apiKey + ":" + .apiSecret)
+          }),
+          "params": {},
+          "files": {}
+        },
+        "metadata": ~r batch.metadata[],
+        "batched": true,
+        "statusCode": 200,
+        "destination": batch[0].destination
+      })[];
+
+  - name: finalPayload
+    template: |
+      [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents]

src/cdk/v2/destinations/bloomreach/utils.ts

@@ -0,0 +1,31 @@
+import { isObject, isEmptyObject, getIntegrationsObj } from '../../../../v0/util';
+import { RudderMessage, Destination } from '../../../../types';
+
+const getCustomerIDsFromIntegrationObject = (message: RudderMessage): any => {
+  const integrationObj = getIntegrationsObj(message, 'bloomreach' as any) || {};
+  const { hardID, softID } = integrationObj;
+  const customerIDs = {};
+
+  if (isObject(hardID) && !isEmptyObject(hardID)) {
+    Object.keys(hardID).forEach((id) => {
+      customerIDs[id] = hardID[id];
+    });
+  }
+
+  if (isObject(softID) && !isEmptyObject(softID)) {
+    Object.keys(softID).forEach((id) => {
+      customerIDs[id] = softID[id];
+    });
+  }
+
+  return customerIDs;
+};
+
+export const prepareCustomerIDs = (message: RudderMessage, destination: Destination): any => {
+  const customerIDs = {
+    [destination.Config.hardID]: message.userId,
+    [destination.Config.softID]: message.anonymousId,
+    ...getCustomerIDsFromIntegrationObject(message),
+  };
+  return customerIDs;
+};

src/cdk/v2/destinations/movable_ink/config.js

@@ -1,3 +1,4 @@
 module.exports = {
-  MAX_REQUEST_SIZE_IN_BYTES: 13500,
+  MAX_REQUEST_SIZE_IN_BYTES: 1000000,
+  MAX_BATCH_SIZE: 1000,
 };

src/cdk/v2/destinations/movable_ink/procWorkflow.yaml

@@ -24,6 +24,7 @@ steps:
       $.assertConfig(.destination.Config.accessKey, "Access key is not present . Aborting");
       $.assertConfig(.destination.Config.accessSecret, "Access Secret is not present. Aborting");
       $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting");
+      $.assert(!(messageType === {{$.EventType.TRACK}} && !(.message.event)), "Event name is not present. Aborting");
 
       const userId = .message.().(
           {{{{$.getGenericPaths("userIdOnly")}}}};

src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml

@@ -42,7 +42,7 @@ steps:
     description: Batches the successfulEvents
     template: |
       let batches = $.BatchUtils.chunkArrayBySizeAndLength(
-      $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES}).items;
+      $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES, maxItems: $.MAX_BATCH_SIZE}).items;
 
       batches@batch.({
         "batchedRequest": {

src/cdk/v2/destinations/ninetailed/config.js

@@ -17,10 +17,6 @@ const ConfigCategories = {
     type: 'identify',
     name: 'identifyMapping',
   },
-  PAGE: {
-    type: 'page',
-    name: 'pageMapping',
-  },
 };
 
 // MAX_BATCH_SIZE : // Maximum number of events to send in a single batch

src/cdk/v2/destinations/ninetailed/procWorkflow.yaml

@@ -16,7 +16,7 @@ steps:
     template: |
       let messageType = $.outputs.messageType;
       $.assert(messageType, "message Type is not present. Aborting");
-      $.assert(messageType in {{$.EventType.([.TRACK,.IDENTIFY,.PAGE])}}, "message type " + messageType + " is not supported");
+      $.assert(messageType in {{$.EventType.([.TRACK,.IDENTIFY])}}, "message type " + messageType + " is not supported");
       $.assertConfig(.destination.Config.organisationId, "Organisation ID is not present. Aborting");
       $.assertConfig(.destination.Config.environment, "Environment is not present. Aborting");
   - name: preparePayload

src/cdk/v2/destinations/ninetailed/utils.js

@@ -31,12 +31,6 @@ const constructFullPayload = (message) => {
         config.mappingConfig[config.ConfigCategories.IDENTIFY.name],
       );
       break;
-    case 'page':
-      typeSpecifcPayload = constructPayload(
-        message,
-        config.mappingConfig[config.ConfigCategories.PAGE.name],
-      );
-      break;
     default:
       break;
   }

src/constants/destinationCanonicalNames.js

@@ -165,6 +165,7 @@ const DestCanonicalNames = {
     'google adwords offline conversions',
   ],
   koala: ['Koala', 'koala', 'KOALA'],
+  bloomreach: ['Bloomreach', 'bloomreach', 'BLOOMREACH'],
 };
 
 module.exports = { DestHandlerMap, DestCanonicalNames };

src/features.json

@@ -69,6 +69,7 @@
     "NINETAILED": true,
     "KOALA": true,
     "LINKEDIN_ADS": true,
+    "BLOOMREACH": true,
     "MOVABLE_INK": true
   },
   "regulations": [

src/v0/destinations/ga4/transform.js

@@ -27,6 +27,7 @@ const {
 const {
   getItemsArray,
   validateEventName,
+  prepareUserConsents,
   removeInvalidParams,
   isReservedEventName,
   getGA4ExclusionList,
@@ -238,6 +239,12 @@ const responseBuilder = (message, { Config }) => {
     rawPayload.user_properties = userProperties;
   }
 
+  // Prepare GA4 consents
+  const consents = prepareUserConsents(message);
+  if (!isEmptyObject(consents)) {
+    rawPayload.consent = consents;
+  }
+
   payload = removeUndefinedAndNullValues(payload);
   rawPayload = { ...rawPayload, events: [payload] };