Merge branch 'develop' into test-refactor.fbPixel

.github/workflows/prepare-for-prod-dt-deploy.yml

@@ -144,3 +144,39 @@ jobs:
           git push -u origin hosted-transformer-$TAG_NAME
 
           gh pr create --fill
+
+      - name: Update helm charts and raise pull request for enterprise customers on dedicated transformers
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+        run: |
+          cd rudder-devops
+          git checkout -b dedicated-transformer-$TAG_NAME
+
+          cd customer-objects
+
+          declare -a enabled_ut_customers=()
+          declare -a sub_directories=('enterprise-us' 'enterprise-eu')
+
+          # identify the customers enabled in sub-directories
+          for directory in "${sub_directories[@]}"; do
+            for f in "./$directory"/*; do
+              [[ -f $f ]] || continue
+
+              enabled="$(yq e '.spec.user_transformer.enabled' $f)"
+              if [ $enabled == "true" ]; then
+                  enabled_ut_customers+=( $f )
+              fi
+            done
+          done
+
+          # bump up the customers version and repository information
+          for customer in "${enabled_ut_customers[@]}"; do
+            yq eval -i ".spec.user_transformer.image.version=\"$TAG_NAME\"" $customer
+            yq eval -i ".spec.user_transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" $customer
+            git add $customer
+          done
+
+          git commit -m "chore: upgrade dedicated transformers to $TAG_NAME"
+          git push -u origin dedicated-transformer-$TAG_NAME
+
+          gh pr create --fill

.github/workflows/prepare-for-prod-rollback.yml

@@ -27,11 +27,14 @@ jobs:
           git config --global user.name "GitHub Actions"
           git config --global user.email "[email protected]"
 
+      - name: Clone Devops Repo
+        run: |
+          git clone https://${{secrets.PAT}}@github.com/rudderlabs/rudder-devops.git
+
       - name: Update Helm Charts and Raise Pull Request
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
         run: |
-          git clone https://${{secrets.PAT}}@github.com/rudderlabs/rudder-devops.git
           cd rudder-devops
           git checkout -b shared-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
 
@@ -57,3 +60,38 @@ jobs:
           git push -u origin shared-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
 
           gh pr create --fill
+
+      - name: Update helm charts and raise pull request for enterprise customers on dedicated transformers
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+        run: |
+          cd rudder-devops
+          git checkout -b dedicated-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
+
+          cd customer-objects
+
+          declare -a enabled_ut_customers=()
+          declare -a sub_directories=('enterprise-us' 'enterprise-eu')
+
+          # identify the customers enabled in sub-directories
+          for directory in "${sub_directories[@]}"; do
+            for f in "./$directory"/*; do
+              [[ -f $f ]] || continue
+
+              enabled="$(yq e '.spec.user_transformer.enabled' $f)"
+              if [ $enabled == "true" ]; then
+                  enabled_ut_customers+=( $f )
+              fi
+            done
+          done
+
+          # bump up the customers version and repository information
+          for customer in "${enabled_ut_customers[@]}"; do
+            yq eval -i ".spec.user_transformer.image.version=\"${{ steps.target-version.outputs.tag_name }}\"" $customer
+            git add $customer
+          done
+
+          git commit -m "chore: rollback dedicated transformers to ${{ steps.target-version.outputs.tag_name }}"
+          git push -u origin dedicated-transformer-rollback-${{ steps.target-version.outputs.tag_name }}
+
+          gh pr create --fill

.nvmrc

@@ -1 +1 @@
-18.19.0
+18.19.1

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.4
-FROM node:18.19.0-alpine3.18 AS base
+FROM node:18.19.1-alpine3.18 AS base
 ENV HUSKY 0
 
 RUN apk update

src/cdk/v2/destinations/algolia/rtWorkflow.yaml

@@ -2,7 +2,6 @@ bindings:
   - path: ../../../../v0/destinations/algolia/config
   - name: handleRtTfSingleEventError
     path: ../../../../v0/util/index
-
 steps:
   - name: validateInput
     template: |
@@ -28,10 +27,14 @@ steps:
       $.outputs.transform#idx.error.(
         $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
       )[]
+
   - name: batchSuccessfulEvents
     description: Batches the successfulEvents
     template: |
-      let batches = $.chunk($.outputs.successfulEvents, $.MAX_BATCH_SIZE);
+      const dontBatchTrueEvents =  $.outputs.successfulEvents{.metadata.dontBatch}[];
+      const dontBatchFalseEvents = $.outputs.successfulEvents{!.metadata.dontBatch}[];
+
+      let batches = [...$.chunk(dontBatchFalseEvents, $.MAX_BATCH_SIZE), ...$.chunk(dontBatchTrueEvents, 1)];
       batches@batch.({
         "batchedRequest": {
           "body": {
@@ -56,6 +59,7 @@ steps:
         "statusCode": 200,
         "destination": batch[0].destination
       })[];
+
   - name: finalPayload
     template: |
       [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents]

src/cdk/v2/destinations/koala/procWorkflow.yaml

@@ -0,0 +1,65 @@
+bindings:
+  - name: EventType
+    path: ../../../../constants
+  - path: ../../bindings/jsontemplate
+  - name: defaultRequestConfig
+    path: ../../../../v0/util
+
+steps:
+  - name: validateInput
+    template: |
+      $.assert(.message.type, "message Type is not present. Aborting message");
+      $.assert(.message.type in {{$.EventType.([.IDENTIFY, .TRACK])}}, 
+        "message type " + .message.type + " is not supported");
+      $.assertConfig(.destination.Config.publicKey, "publicKey is not present. Aborting message");
+      $.context.email = .message.().({{{{$.getGenericPaths("emailOnly")}}}});
+      $.context.ko_profile_id = .message.traits.ko_profile_id ?? .message.context.traits.ko_profile_id ?? .message.properties.ko_profile_id;
+      $.assert($.context.email || $.context.ko_profile_id, "Neither email or ko_profile_id are present on traits. Aborting message");
+  - name: setMessageType
+    template: |
+      $.context.messageType = .message.type.toLowerCase();
+  - name: preparePayloadForIdentify
+    condition: $.context.messageType === {{$.EventType.IDENTIFY}}
+    template: |
+      const traits = .message.traits ?? .message.context.traits ?? {};
+      const koTraits = traits{~['ko_profile_id']}
+      const basePayload = {
+        email: $.context.email, 
+        profile_id: $.context.ko_profile_id,
+        identifies: [{
+          type: $.context.messageType,
+          sent_at: .message.().({{{{$.getGenericPaths("timestamp")}}}}),
+          traits: koTraits 
+        }]
+      };
+
+      $.context.payload = basePayload
+  - name: preparePayloadForTrack
+    condition: $.context.messageType === {{$.EventType.TRACK}}
+    template: |
+      const properties = .message.properties ??  {};
+      const koProperties = properties{~['ko_profile_id']}
+      const basePayload = {
+        ip: .message.context.ip ?? .message.request_ip, 
+        email: $.context.email, 
+        profile_id: $.context.ko_profile_id, 
+        events: [{
+          type: $.context.messageType,
+          event: .message.event,
+          message_id: .message.messageId,
+          sent_at: .message.().({{{{$.getGenericPaths("timestamp")}}}}),
+          properties: koProperties,
+          context: .message.context
+        }]
+      };
+
+      $.context.payload = basePayload
+  - name: buildResponseForProcessTransformation
+    template: |
+      const response = $.defaultRequestConfig();
+      response.body.JSON = $.context.payload;
+      response.endpoint = "https://api2.getkoala.com/web/projects/" + .destination.Config.publicKey + "/batch";
+      response.headers = {
+        "content-type": "application/json"
+      };
+      response

src/cdk/v2/destinations/koala/rtWorkflow.yaml

@@ -0,0 +1,31 @@
+bindings:
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
+
+steps:
+  - name: validateInput
+    template: |
+      $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+  - name: transform
+    externalWorkflow:
+      path: ./procWorkflow.yaml
+    loopOverInput: true
+
+  - name: successfulEvents
+    template: |
+      $.outputs.transform#idx.output.({
+        "batchedRequest": .,
+        "batched": false,
+        "destination": ^[idx].destination,
+        "metadata": ^[idx].metadata[],
+        "statusCode": 200
+      })[]
+  - name: failedEvents
+    template: |
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
+  - name: finalPayload
+    template: |
+      [...$.outputs.failedEvents, ...$.outputs.successfulEvents]

src/constants/destinationCanonicalNames.js

@@ -152,6 +152,7 @@ const DestCanonicalNames = {
     'the trade desk',
   ],
   INTERCOM: ['INTERCOM', 'intercom', 'Intercom'],
+  koala: ['Koala', 'koala', 'KOALA'],
 };
 
 module.exports = { DestHandlerMap, DestCanonicalNames };

src/features.json

@@ -66,7 +66,8 @@
     "REDDIT": true,
     "THE_TRADE_DESK": true,
     "INTERCOM": true,
-    "NINETAILED": true
+    "NINETAILED": true,
+    "KOALA": true
   },
   "regulations": [
     "BRAZE",

src/v1/destinations/algolia/networkHandler.js

@@ -0,0 +1,84 @@
+/* eslint-disable no-restricted-syntax */
+const { TransformerProxyError } = require('../../../v0/util/errorTypes');
+const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network');
+const { isHttpStatusSuccess, getAuthErrCategoryFromStCode } = require('../../../v0/util/index');
+
+const {
+  processAxiosResponse,
+  getDynamicErrorType,
+} = require('../../../adapters/utils/networkUtils');
+const tags = require('../../../v0/util/tags');
+
+const responseHandler = (responseParams) => {
+  const { destinationResponse, rudderJobMetadata } = responseParams;
+  const message = `[ALGOLIA Response V1 Handler] - Request Processed Successfully`;
+  const responseWithIndividualEvents = [];
+  // response:
+  // {status: 200, message: 'OK'}
+  // {response:'[ENOTFOUND] :: DNS lookup failed', status: 400}
+  // destinationResponse = {
+  //   response: {"status": 422, "message": "EventType must be one of \"click\", \"conversion\" or \"view\""}, status: 422
+  // }
+  const { response, status } = destinationResponse;
+
+  if (isHttpStatusSuccess(status)) {
+    for (const mData of rudderJobMetadata) {
+      const proxyOutputObj = {
+        statusCode: 200,
+        metadata: mData,
+        error: 'success',
+      };
+      responseWithIndividualEvents.push(proxyOutputObj);
+    }
+
+    return {
+      status,
+      message,
+      destinationResponse,
+      response: responseWithIndividualEvents,
+    };
+  }
+
+  // in case of non 2xx status sending 500 for every event, populate response and update dontBatch to true
+  const errorMessage = response?.error?.message || response?.message || 'unknown error format';
+  let serverStatus = 400;
+  for (const metadata of rudderJobMetadata) {
+    // handling case if dontBatch is true, and again we got invalid from destination
+    if (metadata.dontBatch && status === 422) {
+      responseWithIndividualEvents.push({
+        statusCode: 400,
+        metadata,
+        error: errorMessage,
+      });
+    } else {
+      serverStatus = 500;
+      metadata.dontBatch = true;
+      responseWithIndividualEvents.push({
+        statusCode: 500,
+        metadata,
+        error: errorMessage,
+      });
+    }
+  }
+
+  // sending back 500 for retry
+  throw new TransformerProxyError(
+    `ALGOLIA: Error transformer proxy v1 during ALGOLIA response transformation`,
+    serverStatus,
+    {
+      [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status),
+    },
+    destinationResponse,
+    getAuthErrCategoryFromStCode(status),
+    responseWithIndividualEvents,
+  );
+};
+
+function networkHandler() {
+  this.prepareProxy = prepareProxyRequest;
+  this.proxy = proxyRequest;
+  this.processAxiosResponse = processAxiosResponse;
+  this.responseHandler = responseHandler;
+}
+
+module.exports = { networkHandler };

test/integrations/common/network.ts

@@ -81,4 +81,15 @@ export const networkCallsData = [
       status: 429,
     },
   },
+  {
+    description: 'Mock response depicting DNS lookup failure error',
+    httpReq: {
+      method: 'post',
+      url: 'https://random_test_url/dns_lookup_failure',
+    },
+    httpRes: {
+      data: {},
+      status: 400,
+    },
+  },
 ];