Merge branch 'develop' into feat.tiktok-pinterest-clickId

.github/workflows/create-hotfix-branch.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     # Only allow these users to create new hotfix branch from 'main'
-    if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' ||  github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116')
+    if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'ujjwal-ab') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' ||  github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'ujjwal-ab)
     steps:
       - name: Create Branch
         uses: peterjgrainger/[email protected]

.github/workflows/draft-new-release.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
 
     # Only allow release stakeholders to initiate releases
-    if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823'  || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292') &&  (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292')
+    if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823'  || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'ujjwal-ab') &&  (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'ujjwal-ab')
     steps:
       - name: Checkout
         uses: actions/[email protected]

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.49.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.48.0...v1.49.0) (2023-11-06)
+
+
+### Features
+
+* add new destination tiktok_audience ([#2710](https://github.com/rudderlabs/rudder-transformer/issues/2710)) ([9bc0fd8](https://github.com/rudderlabs/rudder-transformer/commit/9bc0fd8efcee44871a190bd6cb9e89c5cf035ff8))
+* onboard one signal to router transform ([#2785](https://github.com/rudderlabs/rudder-transformer/issues/2785)) ([818858e](https://github.com/rudderlabs/rudder-transformer/commit/818858e046ce5f9735bbb97715c43a959ad3aa3c))
+* onboard revenuecat as a source ([#2774](https://github.com/rudderlabs/rudder-transformer/issues/2774)) ([55f9637](https://github.com/rudderlabs/rudder-transformer/commit/55f96374b4d73db7013c1d5e72bfc9c8257b224b))
+
+
+### Bug Fixes
+
+* add check to remove null and undefined properties before sending ([#2796](https://github.com/rudderlabs/rudder-transformer/issues/2796)) ([6e89cd3](https://github.com/rudderlabs/rudder-transformer/commit/6e89cd3f67ea887ba17c1cd5ffbca6675f54d96c))
+* allow support for full url from UI in freshsales and freshmarketer ([#2780](https://github.com/rudderlabs/rudder-transformer/issues/2780)) ([570532c](https://github.com/rudderlabs/rudder-transformer/commit/570532ce790c05a69621d9289758a1b1a7acda8c))
+* busgnag issues for klaviyo, freshsales, customeio  ([#2795](https://github.com/rudderlabs/rudder-transformer/issues/2795)) ([11fb7c4](https://github.com/rudderlabs/rudder-transformer/commit/11fb7c47910681833e37d25a1573d2005e62742b))
+
+## [1.48.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.47.0...v1.48.0) (2023-11-02)
+
+
+### Features
+
+* add support to add custom network policies for specific workspaces in faas pods ([bc1a760](https://github.com/rudderlabs/rudder-transformer/commit/bc1a76066c0aeb43776ded0b266ec48f5e69aa16))
+
 ## [1.47.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.46.5...v1.47.0) (2023-10-30)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.47.0",
+  "version": "1.49.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -67,10 +67,10 @@
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",
     "ajv-formats": "^2.1.1",
-    "axios": "^1.4.0",
+    "axios": "^1.6.0",
     "btoa": "^1.2.1",
     "component-each": "^0.2.6",
-    "crypto-js": "^4.1.1",
+    "crypto-js": "^4.2.0",
     "dotenv": "^16.0.3",
     "flat": "^5.0.2",
     "form-data": "^4.0.0",

src/features.json

@@ -58,7 +58,8 @@
     "OPTIMIZELY_FULLSTACK": true,
     "TWITTER_ADS": true,
     "CLEVERTAP": true,
-    "TIKTOK_AUDIENCE": true,
-    "ORTTO": true
+    "ORTTO": true,
+    "ONE_SIGNAL": true,
+    "TIKTOK_AUDIENCE": true
   }
 }

src/legacy/router.js

@@ -571,14 +571,9 @@ if (startDestTransformer) {
             destEvents.length > 0 && destEvents[0].metadata
               ? getMetadata(destEvents[0].metadata)
               : {};
-          const userFuncStartTime = new Date();
           if (transformationVersionId) {
             let destTransformedEvents;
             try {
-              stats.counter('user_transform_function_input_events', destEvents.length, {
-                processSessions,
-                ...metaTags,
-              });
               destTransformedEvents = await userTransformHandler()(
                 destEvents,
                 transformationVersionId,
@@ -630,12 +625,6 @@ if (startDestTransformer) {
                 processSessions,
                 ...metaTags,
               });
-            } finally {
-              stats.timing('user_transform_function_latency', userFuncStartTime, {
-                transformationVersionId,
-                processSessions,
-                ...metaTags,
-              });
             }
           } else {
             const errorMessage = 'Transformation VersionID not found';

src/services/userTransform.ts

@@ -85,9 +85,6 @@ export default class UserTransformService {
         }
         const userFuncStartTime = new Date();
         try {
-          stats.counter('user_transform_function_input_events', eventsToProcess.length, {
-            ...metaTags,
-          });
           const destTransformedEvents: UserTransformationResponse[] = await userTransformHandler()(
             eventsToProcess,
             transformationVersionId,

src/util/customTransformer-faas.js

@@ -1,7 +1,7 @@
 const { v4: uuidv4 } = require('uuid');
 const crypto = require('crypto');
 const NodeCache = require('node-cache');
-const { getMetadata } = require('../v0/util');
+const { getMetadata, getTransformationMetadata } = require('../v0/util');
 const stats = require('./stats');
 const {
   setupFaasFunction,
@@ -24,8 +24,8 @@ function generateFunctionName(userTransformation, libraryVersionIds, testMode) {
   const ids = [userTransformation.workspaceId, userTransformation.versionId].concat(
     (libraryVersionIds || []).sort(),
   );
-  const hash = crypto.createHash('md5').update(`${ids}`).digest('hex');
 
+  const hash = crypto.createHash('md5').update(`${ids}`).digest('hex');
   return `fn-${userTransformation.workspaceId}-${hash}`.substring(0, 63).toLowerCase();
 }
 
@@ -82,10 +82,10 @@ async function setOpenFaasUserTransform(
   libraryVersionIds,
   pregeneratedFnName,
   testMode = false,
+  trMetadata = {},
 ) {
   const tags = {
     transformerVersionId: userTransformation.versionId,
-    language: userTransformation.language,
     identifier: 'openfaas',
     testMode,
   };
@@ -106,6 +106,7 @@ async function setOpenFaasUserTransform(
       testMode,
     ),
     testMode,
+    trMetadata,
   );
 
   stats.timing('creation_time', setupTime, tags);
@@ -126,24 +127,21 @@ async function runOpenFaasUserTransform(
   if (events.length === 0) {
     throw new Error('Invalid payload. No events');
   }
-  const metaTags = events[0].metadata ? getMetadata(events[0].metadata) : {};
-  const tags = {
-    transformerVersionId: userTransformation.versionId,
-    language: userTransformation.language,
-    identifier: 'openfaas',
-    testMode,
-    ...metaTags,
-  };
 
+  const trMetadata = events[0].metadata ? getTransformationMetadata(events[0].metadata) : {};
   // check and deploy faas function if not exists
   const functionName = generateFunctionName(userTransformation, libraryVersionIds, testMode);
   if (testMode) {
-    await setOpenFaasUserTransform(userTransformation, libraryVersionIds, functionName, testMode);
+    await setOpenFaasUserTransform(
+      userTransformation,
+      libraryVersionIds,
+      functionName,
+      testMode,
+      trMetadata,
+    );
   }
 
-  const invokeTime = new Date();
-  stats.counter('events_to_process', events.length, tags);
-  const result = await executeFaasFunction(
+  return await executeFaasFunction(
     functionName,
     events,
     userTransformation.versionId,
@@ -156,9 +154,8 @@ async function runOpenFaasUserTransform(
       testMode,
     ),
     testMode,
+    trMetadata,
   );
-  stats.timing('run_time', invokeTime, tags);
-  return result;
 }
 
 module.exports = {

src/util/customTransformer-v1.js

@@ -1,7 +1,7 @@
 const ivm = require('isolated-vm');
 
 const { getFactory } = require('./ivmFactory');
-const { getMetadata } = require('../v0/util');
+const { getMetadata, getTransformationMetadata } = require('../v0/util');
 const logger = require('../logger');
 const stats = require('./stats');
 
@@ -54,63 +54,48 @@ async function userTransformHandlerV1(
   libraryVersionIds,
   testMode = false,
 ) {
-  /*
-  Removing pool usage to address memory leaks
-  Env variable ON_DEMAND_ISOLATE_VM is not being used anymore
-  */
-  if (userTransformation.versionId) {
-    const metaTags = events.length && events[0].metadata ? getMetadata(events[0].metadata) : {};
-    const tags = {
-      transformerVersionId: userTransformation.versionId,
-      identifier: 'v1',
-      ...metaTags,
-    };
-
-    logger.debug(`Isolate VM being created... `);
-    const isolatevmFactory = await getFactory(
-      userTransformation.code,
-      libraryVersionIds,
-      userTransformation.versionId,
-      userTransformation.secrets || {},
-      testMode,
-    );
-    const isolatevm = await isolatevmFactory.create();
-    logger.debug(`Isolate VM created... `);
+  if (!userTransformation.versionId) {
+    return { transformedEvents : events };
+  }
 
-    // Transform the event...
-    stats.counter('events_to_process', events.length, tags);
-    const isolateStartWallTime = calculateMsFromIvmTime(isolatevm.isolateStartWallTime);
-    const isolateStartCPUTime = calculateMsFromIvmTime(isolatevm.isolateStartCPUTime);
+  const isolatevmFactory = await getFactory(
+    userTransformation.code,
+    libraryVersionIds,
+    userTransformation.versionId,
+    userTransformation.secrets || {},
+    testMode,
+  );
 
-    const invokeTime = new Date();
-    let transformedEvents;
-    // Destroy isolatevm in case of execution errors
-    try {
-      transformedEvents = await transform(isolatevm, events);
-    } catch (err) {
-      logger.error(`Error encountered while executing transformation: ${err.message}`);
-      isolatevmFactory.destroy(isolatevm);
-      throw err;
-    }
-    const { logs } = isolatevm;
-    stats.timing('run_time', invokeTime, tags);
-    const isolateEndWallTime = calculateMsFromIvmTime(isolatevm.isolate.wallTime);
-    const isolateEndCPUTime = calculateMsFromIvmTime(isolatevm.isolate.cpuTime);
+  logger.debug(`Creating IsolateVM`);
+  const isolatevm = await isolatevmFactory.create();
 
-    //TODO: fix "Value is not a valid number: NaN" error and uncomment
-    //stats.timing('isolate_wall_time', isolateEndWallTime - isolateStartWallTime, tags);
-    //stats.timing('isolate_cpu_time', isolateEndCPUTime - isolateStartCPUTime, tags);
+  const invokeTime = new Date();
+  let transformedEvents;
+  let logs;
+  let transformationError;
 
-    // Destroy the isolated vm resources created
-    logger.debug(`Isolate VM being destroyed... `);
+  try {
+    transformedEvents = await transform(isolatevm, events);
+    logs = isolatevm.logs;
+  } catch (err) {
+    logger.error(`Error encountered while executing transformation: ${err.message}`);
+    transformationError = err;
+    throw err;
+  } finally {
+    logger.debug(`Destroying IsolateVM`);
     isolatevmFactory.destroy(isolatevm);
-    logger.debug(`Isolate VM destroyed... `);
-
-    return { transformedEvents, logs };
-    // Events contain message and destination. We take the message part of event and run transformation on it.
-    // And put back the destination after transforrmation
+    // send the observability stats
+    const tags = {
+      identifier: 'v1',
+      errored: transformationError ? true : false,
+      ...events.length && events[0].metadata ? getMetadata(events[0].metadata) : {},
+      ...events.length && events[0].metadata ? getTransformationMetadata(events[0].metadata) : {}
+    }
+    stats.counter('user_transform_function_input_events', events.length, tags);
+    stats.timing('user_transform_function_latency', invokeTime, tags);
   }
-  return { transformedEvents: events };
+
+  return { transformedEvents, logs };
 }
 
 async function setUserTransformHandlerV1() {