Merge branch 'develop' into reddit.refactor-proxy

CONTRIBUTING.md

@@ -35,7 +35,6 @@ See the project's [README](README.md) for further information about working in t
    - Include instructions on how to test your changes.
 3. Your branch may be merged once all configured checks pass, including:
    - A review from appropriate maintainers
-4. Along with the PR in transformer raise a PR against [config-generator][config-generator] with the configurations.
 
 ## Committing
 

package.json

@@ -66,6 +66,7 @@
     "@pyroscope/nodejs": "^0.2.6",
     "@rudderstack/integrations-lib": "^0.2.4",
     "@rudderstack/workflow-engine": "^0.7.2",
+    "@shopify/jest-koa-mocks": "^5.1.1",
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",
     "ajv-formats": "^2.1.1",

src/cdk/v2/destinations/algolia/procWorkflow.yaml

@@ -61,7 +61,7 @@ steps:
       const filters = $.context.payload.filters;
       const objectIDs = $.context.payload.objectIDs;
       $.assert(!(filters && objectIDs), "event can't have both objectIds and filters at the same time.");
-      $.assert(filters || objectIDs, "Either filters or  objectIds is required.");
+      $.assert(filters.length || objectIDs.length, "Either filters or objectIds is required and must be non empty.");
 
   - name: validatePayloadForClickEvent
     condition: $.context.payload.eventType === "click"

src/cdk/v2/destinations/ninetailed/config.js

@@ -0,0 +1,31 @@
+const { getMappingConfig } = require('../../../../v0/util');
+
+const ConfigCategories = {
+  GENERAL: {
+    type: 'general',
+    name: 'generalPayloadMapping',
+  },
+  CONTEXT: {
+    type: 'context',
+    name: 'contextMapping',
+  },
+  TRACK: {
+    type: 'track',
+    name: 'trackMapping',
+  },
+  IDENTIFY: {
+    type: 'identify',
+    name: 'identifyMapping',
+  },
+  PAGE: {
+    type: 'page',
+    name: 'pageMapping',
+  },
+};
+
+// MAX_BATCH_SIZE : // Maximum number of events to send in a single batch
+const mappingConfig = getMappingConfig(ConfigCategories, __dirname);
+const batchEndpoint =
+  'https://experience.ninetailed.co/v2/organizations/{{organisationId}}/environments/{{environment}}/events';
+
+module.exports = { ConfigCategories, mappingConfig, batchEndpoint, MAX_BATCH_SIZE: 200 };

src/cdk/v2/destinations/ninetailed/data/contextMapping.json

@@ -0,0 +1,43 @@
+[
+  {
+    "sourceKeys": "app.name",
+    "required": true,
+    "destKey": "app.name"
+  },
+  {
+    "sourceKeys": "app.version",
+    "required": true,
+    "destKey": "app.version"
+  },
+  {
+    "sourceKeys": "campaign",
+    "destKey": "campaign"
+  },
+  {
+    "sourceKeys": "library.name",
+    "required": true,
+    "destKey": "library.name"
+  },
+  {
+    "sourceKeys": "library.version",
+    "required": true,
+    "destKey": "library.version"
+  },
+  {
+    "sourceKeys": "locale",
+    "destKey": "locale"
+  },
+  {
+    "sourceKeys": "page",
+    "destKey": "page"
+  },
+  {
+    "sourceKeys": "userAgent",
+    "destKey": "userAgent"
+  },
+  {
+    "sourceKeys": "location",
+    "required": true,
+    "destKey": "location"
+  }
+]

src/cdk/v2/destinations/ninetailed/data/generalPayloadMapping.json

@@ -0,0 +1,25 @@
+[
+  {
+    "sourceKeys": "anonymousId",
+    "required": true,
+    "destKey": "anonymousId"
+  },
+  {
+    "sourceKeys": "messageId",
+    "required": true,
+    "destKey": "messageId"
+  },
+  {
+    "sourceKeys": "channel",
+    "required": true,
+    "destKey": "channel"
+  },
+  {
+    "sourceKeys": "type",
+    "destKey": "type"
+  },
+  {
+    "sourceKeys": "originalTimestamp",
+    "destKey": "originalTimestamp"
+  }
+]

src/cdk/v2/destinations/ninetailed/data/identifyMapping.json

@@ -0,0 +1,14 @@
+[
+  {
+    "sourceKeys": "traits",
+    "sourceFromGenericMap": true,
+    "required": true,
+    "destKey": "traits"
+  },
+  {
+    "sourceKeys": "userIdOnly",
+    "sourceFromGenericMap": true,
+    "required": true,
+    "destKey": "userId"
+  }
+]

src/cdk/v2/destinations/ninetailed/data/pageMapping.json

@@ -0,0 +1,7 @@
+[
+  {
+    "sourceKeys": "properties",
+    "required": true,
+    "destKey": "properties"
+  }
+]

src/cdk/v2/destinations/ninetailed/data/trackMapping.json

@@ -0,0 +1,12 @@
+[
+  {
+    "sourceKeys": "properties",
+    "required": true,
+    "destKey": "properties"
+  },
+  {
+    "sourceKeys": "event",
+    "required": true,
+    "destKey": "event"
+  }
+]

src/cdk/v2/destinations/ninetailed/procWorkflow.yaml

@@ -0,0 +1,33 @@
+bindings:
+  - name: EventType
+    path: ../../../../constants
+  - path: ../../bindings/jsontemplate
+  - name: defaultRequestConfig
+    path: ../../../../v0/util
+  - name: removeUndefinedAndNullValues
+    path: ../../../../v0/util
+  - path: ./utils
+
+steps:
+  - name: messageType
+    template: |
+      .message.type.toLowerCase();
+  - name: validateInput
+    template: |
+      let messageType = $.outputs.messageType;
+      $.assert(messageType, "message Type is not present. Aborting");
+      $.assert(messageType in {{$.EventType.([.TRACK,.IDENTIFY,.PAGE])}}, "message type " + messageType + " is not supported");
+      $.assertConfig(.destination.Config.organisationId, "Organisation ID is not present. Aborting");
+      $.assertConfig(.destination.Config.environment, "Environment is not present. Aborting");
+  - name: preparePayload
+    template: |
+      const payload = $.constructFullPayload(.message);
+      $.context.payload = $.removeUndefinedAndNullValues(payload);
+
+  - name: buildResponse
+    template: |
+      const response = $.defaultRequestConfig();
+      response.body.JSON.events = [$.context.payload];
+      response.endpoint = $.getEndpoint(.destination.Config);
+      response.method = "POST";
+      response

src/cdk/v2/destinations/ninetailed/rtWorkflow.yaml

@@ -0,0 +1,35 @@
+bindings:
+  - path: ./config
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
+  - path: ./utils
+steps:
+  - name: validateInput
+    template: |
+      $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+  - name: transform
+    externalWorkflow:
+      path: ./procWorkflow.yaml
+    loopOverInput: true
+
+  - name: successfulEvents
+    template: |
+      $.outputs.transform#idx.output.({
+        "output": .body.JSON.events[0],
+        "destination": ^[idx].destination,
+        "metadata": ^[idx].metadata
+      })[]
+  - name: failedEvents
+    template: |
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
+  - name: batchSuccessfulEvents
+    description: Batches the successfulEvents
+    template: |
+      $.batchResponseBuilder($.outputs.successfulEvents);
+
+  - name: finalPayload
+    template: |
+      [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents]