Merge branch 'develop' into feat.slack-source

src/cdk/v2/destinations/algolia/procWorkflow.yaml

@@ -61,7 +61,7 @@ steps:
       const filters = $.context.payload.filters;
       const objectIDs = $.context.payload.objectIDs;
       $.assert(!(filters && objectIDs), "event can't have both objectIds and filters at the same time.");
-      $.assert(filters || objectIDs, "Either filters or  objectIds is required.");
+      $.assert(filters.length || objectIDs.length, "Either filters or objectIds is required and must be non empty.");
 
   - name: validatePayloadForClickEvent
     condition: $.context.payload.eventType === "click"

src/cdk/v2/destinations/ninetailed/config.js

@@ -0,0 +1,31 @@
+const { getMappingConfig } = require('../../../../v0/util');
+
+const ConfigCategories = {
+  GENERAL: {
+    type: 'general',
+    name: 'generalPayloadMapping',
+  },
+  CONTEXT: {
+    type: 'context',
+    name: 'contextMapping',
+  },
+  TRACK: {
+    type: 'track',
+    name: 'trackMapping',
+  },
+  IDENTIFY: {
+    type: 'identify',
+    name: 'identifyMapping',
+  },
+  PAGE: {
+    type: 'page',
+    name: 'pageMapping',
+  },
+};
+
+// MAX_BATCH_SIZE : // Maximum number of events to send in a single batch
+const mappingConfig = getMappingConfig(ConfigCategories, __dirname);
+const batchEndpoint =
+  'https://experience.ninetailed.co/v2/organizations/{{organisationId}}/environments/{{environment}}/events';
+
+module.exports = { ConfigCategories, mappingConfig, batchEndpoint, MAX_BATCH_SIZE: 200 };

src/cdk/v2/destinations/ninetailed/data/contextMapping.json

@@ -0,0 +1,43 @@
+[
+  {
+    "sourceKeys": "app.name",
+    "required": true,
+    "destKey": "app.name"
+  },
+  {
+    "sourceKeys": "app.version",
+    "required": true,
+    "destKey": "app.version"
+  },
+  {
+    "sourceKeys": "campaign",
+    "destKey": "campaign"
+  },
+  {
+    "sourceKeys": "library.name",
+    "required": true,
+    "destKey": "library.name"
+  },
+  {
+    "sourceKeys": "library.version",
+    "required": true,
+    "destKey": "library.version"
+  },
+  {
+    "sourceKeys": "locale",
+    "destKey": "locale"
+  },
+  {
+    "sourceKeys": "page",
+    "destKey": "page"
+  },
+  {
+    "sourceKeys": "userAgent",
+    "destKey": "userAgent"
+  },
+  {
+    "sourceKeys": "location",
+    "required": true,
+    "destKey": "location"
+  }
+]

src/cdk/v2/destinations/ninetailed/data/generalPayloadMapping.json

@@ -0,0 +1,25 @@
+[
+  {
+    "sourceKeys": "anonymousId",
+    "required": true,
+    "destKey": "anonymousId"
+  },
+  {
+    "sourceKeys": "messageId",
+    "required": true,
+    "destKey": "messageId"
+  },
+  {
+    "sourceKeys": "channel",
+    "required": true,
+    "destKey": "channel"
+  },
+  {
+    "sourceKeys": "type",
+    "destKey": "type"
+  },
+  {
+    "sourceKeys": "originalTimestamp",
+    "destKey": "originalTimestamp"
+  }
+]

src/cdk/v2/destinations/ninetailed/data/identifyMapping.json

@@ -0,0 +1,14 @@
+[
+  {
+    "sourceKeys": "traits",
+    "sourceFromGenericMap": true,
+    "required": true,
+    "destKey": "traits"
+  },
+  {
+    "sourceKeys": "userIdOnly",
+    "sourceFromGenericMap": true,
+    "required": true,
+    "destKey": "userId"
+  }
+]

src/cdk/v2/destinations/ninetailed/data/pageMapping.json

@@ -0,0 +1,7 @@
+[
+  {
+    "sourceKeys": "properties",
+    "required": true,
+    "destKey": "properties"
+  }
+]

src/cdk/v2/destinations/ninetailed/data/trackMapping.json

@@ -0,0 +1,12 @@
+[
+  {
+    "sourceKeys": "properties",
+    "required": true,
+    "destKey": "properties"
+  },
+  {
+    "sourceKeys": "event",
+    "required": true,
+    "destKey": "event"
+  }
+]

src/cdk/v2/destinations/ninetailed/procWorkflow.yaml

@@ -0,0 +1,33 @@
+bindings:
+  - name: EventType
+    path: ../../../../constants
+  - path: ../../bindings/jsontemplate
+  - name: defaultRequestConfig
+    path: ../../../../v0/util
+  - name: removeUndefinedAndNullValues
+    path: ../../../../v0/util
+  - path: ./utils
+
+steps:
+  - name: messageType
+    template: |
+      .message.type.toLowerCase();
+  - name: validateInput
+    template: |
+      let messageType = $.outputs.messageType;
+      $.assert(messageType, "message Type is not present. Aborting");
+      $.assert(messageType in {{$.EventType.([.TRACK,.IDENTIFY,.PAGE])}}, "message type " + messageType + " is not supported");
+      $.assertConfig(.destination.Config.organisationId, "Organisation ID is not present. Aborting");
+      $.assertConfig(.destination.Config.environment, "Environment is not present. Aborting");
+  - name: preparePayload
+    template: |
+      const payload = $.constructFullPayload(.message);
+      $.context.payload = $.removeUndefinedAndNullValues(payload);
+
+  - name: buildResponse
+    template: |
+      const response = $.defaultRequestConfig();
+      response.body.JSON.events = [$.context.payload];
+      response.endpoint = $.getEndpoint(.destination.Config);
+      response.method = "POST";
+      response

src/cdk/v2/destinations/ninetailed/rtWorkflow.yaml

@@ -0,0 +1,35 @@
+bindings:
+  - path: ./config
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
+  - path: ./utils
+steps:
+  - name: validateInput
+    template: |
+      $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+  - name: transform
+    externalWorkflow:
+      path: ./procWorkflow.yaml
+    loopOverInput: true
+
+  - name: successfulEvents
+    template: |
+      $.outputs.transform#idx.output.({
+        "output": .body.JSON.events[0],
+        "destination": ^[idx].destination,
+        "metadata": ^[idx].metadata
+      })[]
+  - name: failedEvents
+    template: |
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
+  - name: batchSuccessfulEvents
+    description: Batches the successfulEvents
+    template: |
+      $.batchResponseBuilder($.outputs.successfulEvents);
+
+  - name: finalPayload
+    template: |
+      [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents]

src/cdk/v2/destinations/ninetailed/utils.js

@@ -0,0 +1,109 @@
+const { BatchUtils } = require('@rudderstack/workflow-engine');
+const config = require('./config');
+const { constructPayload } = require('../../../../v0/util');
+
+/**
+ * This fucntion constructs payloads based upon mappingConfig for all calls
+ * We build context as it has some specific payloads with default values so just breaking them down
+ * @param {*} message
+ * @returns
+ */
+const constructFullPayload = (message) => {
+  const context = constructPayload(
+    message?.context || {},
+    config.mappingConfig[config.ConfigCategories.CONTEXT.name],
+  );
+  const payload = constructPayload(
+    message,
+    config.mappingConfig[config.ConfigCategories.GENERAL.name],
+  );
+  let typeSpecifcPayload;
+  switch (message.type) {
+    case 'track':
+      typeSpecifcPayload = constructPayload(
+        message,
+        config.mappingConfig[config.ConfigCategories.TRACK.name],
+      );
+      break;
+    case 'identify':
+      typeSpecifcPayload = constructPayload(
+        message,
+        config.mappingConfig[config.ConfigCategories.IDENTIFY.name],
+      );
+      break;
+    case 'page':
+      typeSpecifcPayload = constructPayload(
+        message,
+        config.mappingConfig[config.ConfigCategories.PAGE.name],
+      );
+      break;
+    default:
+      break;
+  }
+  payload.context = context;
+  return { ...payload, ...typeSpecifcPayload }; // merge base and type-specific payloads;
+};
+
+const getEndpoint = (Config) => {
+  const { organisationId, environment } = Config;
+  return config.batchEndpoint
+    .replace('{{organisationId}}', organisationId)
+    .replace('{{environment}}', environment);
+};
+
+const mergeMetadata = (batch) => {
+  const metadata = [];
+  batch.forEach((event) => {
+    metadata.push(event.metadata);
+  });
+  return metadata;
+};
+
+const getMergedEvents = (batch) => {
+  const events = [];
+  batch.forEach((event) => {
+    events.push(event.output);
+  });
+  return events;
+};
+
+const batchBuilder = (batch) => ({
+  batchedRequest: {
+    body: {
+      JSON: { events: getMergedEvents(batch) },
+      JSON_ARRAY: {},
+      XML: {},
+      FORM: {},
+    },
+    version: '1',
+    type: 'REST',
+    method: 'POST',
+    endpoint: getEndpoint(batch[0].destination.Config),
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    params: {},
+    files: {},
+  },
+  metadata: mergeMetadata(batch),
+  batched: true,
+  statusCode: 200,
+  destination: batch[0].destination,
+});
+
+/**
+ * This fucntions make chunk of successful events based on MAX_BATCH_SIZE
+ * and then build the response for each chunk to be returned as object of an array
+ * @param {*} events
+ * @returns
+ */
+const batchResponseBuilder = (events) => {
+  const batches = BatchUtils.chunkArrayBySizeAndLength(events, { maxItems: config.MAX_BATCH_SIZE });
+  const response = [];
+  batches.items.forEach((batch) => {
+    response.push(batchBuilder(batch));
+  });
+  return response;
+};
+
+module.exports = { constructFullPayload, getEndpoint, batchResponseBuilder };

src/features.json

@@ -65,7 +65,8 @@
     "TIKTOK_AUDIENCE": true,
     "REDDIT": true,
     "THE_TRADE_DESK": true,
-    "INTERCOM": true
+    "INTERCOM": true,
+    "NINETAILED": true
   },
   "regulations": [
     "BRAZE",

src/v0/destinations/adobe_analytics/transform.js

@@ -18,6 +18,7 @@ const {
   getIntegrationsObj,
   removeUndefinedAndNullValues,
   simpleProcessRouterDest,
+  validateEventAndLowerCaseConversion,
 } = require('../../util');
 
 const {
@@ -307,7 +308,7 @@ const processTrackEvent = (message, adobeEventName, destinationConfig, extras =
     destinationConfig;
   const { event: rawMessageEvent, properties } = message;
   const { overrideEventString, overrideProductString } = properties;
-  const event = rawMessageEvent.toLowerCase();
+  const event = validateEventAndLowerCaseConversion(rawMessageEvent, true, true);
   const adobeEventArr = adobeEventName ? adobeEventName.split(',') : [];
   // adobeEventArr is an array of events which is defined as
   // ["eventName", "mapped Adobe Event=mapped merchproperty's value", "mapped Adobe Event=mapped merchproperty's value", . . .]

src/v0/destinations/gainsight_px/util.js

@@ -6,13 +6,13 @@ const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils');
 const { JSON_MIME_TYPE } = require('../../util/constant');
 
 const handleErrorResponse = (error, customErrMessage, expectedErrStatus, defaultStatus = 400) => {
+  let destResp;
   let errMessage = '';
   let errorStatus = defaultStatus;
 
   if (error.response && error.response.data) {
-    errMessage = error.response.data.externalapierror
-      ? JSON.stringify(error.response.data.externalapierror)
-      : JSON.stringify(error.response.data);
+    destResp = error.response?.data?.externalapierror ?? error.response?.data;
+    errMessage = JSON.stringify(destResp);
 
     errorStatus = error.response.status;
 
@@ -26,7 +26,7 @@ const handleErrorResponse = (error, customErrMessage, expectedErrStatus, default
     {
       [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(errorStatus),
     },
-    error,
+    destResp,
   );
 };
 

src/v0/destinations/google_adwords_remarketing_lists/transform.js

@@ -218,7 +218,7 @@ const processEvent = async (metadata, message, destination) => {
     }
 
     Object.values(createdPayload).forEach((data) => {
-      const consentObj = populateConsentForGoogleDestinations(message.properties);
+      const consentObj = populateConsentForGoogleDestinations(destination.Config);
       response.push(responseBuilder(metadata, data, destination, message, consentObj));
     });
     return response;