Merge branch 'develop' into feat.salesforce-auth

.github/workflows/create-hotfix-branch.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     # Only allow these users to create new hotfix branch from 'main'
-    if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' ||  github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116')
+    if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'ujjwal-ab') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' ||  github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'ujjwal-ab)
     steps:
       - name: Create Branch
         uses: peterjgrainger/[email protected]

.github/workflows/draft-new-release.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
 
     # Only allow release stakeholders to initiate releases
-    if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823'  || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292') &&  (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292')
+    if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823'  || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'ujjwal-ab') &&  (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'ujjwal-ab')
     steps:
       - name: Checkout
         uses: actions/[email protected]

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.48.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.47.0...v1.48.0) (2023-11-02)
+
+
+### Features
+
+* add support to add custom network policies for specific workspaces in faas pods ([bc1a760](https://github.com/rudderlabs/rudder-transformer/commit/bc1a76066c0aeb43776ded0b266ec48f5e69aa16))
+
 ## [1.47.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.46.5...v1.47.0) (2023-10-30)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.47.0",
+  "version": "1.48.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/features.json

@@ -59,7 +59,8 @@
     "OPTIMIZELY_FULLSTACK": true,
     "TWITTER_ADS": true,
     "CLEVERTAP": true,
-    "TIKTOK_AUDIENCE": true,
-    "ORTTO": true
+    "ORTTO": true,
+    "ONE_SIGNAL": true,
+    "TIKTOK_AUDIENCE": true
   }
 }

src/util/customTransformer-faas.js

@@ -1,7 +1,7 @@
 const { v4: uuidv4 } = require('uuid');
 const crypto = require('crypto');
 const NodeCache = require('node-cache');
-const { getMetadata } = require('../v0/util');
+const { getMetadata, getTransformationMetadata } = require('../v0/util');
 const stats = require('./stats');
 const {
   setupFaasFunction,
@@ -82,10 +82,10 @@ async function setOpenFaasUserTransform(
   libraryVersionIds,
   pregeneratedFnName,
   testMode = false,
+  trMetadata = {},
 ) {
   const tags = {
     transformerVersionId: userTransformation.versionId,
-    language: userTransformation.language,
     identifier: 'openfaas',
     testMode,
   };
@@ -106,6 +106,7 @@ async function setOpenFaasUserTransform(
       testMode,
     ),
     testMode,
+    trMetadata,
   );
 
   stats.timing('creation_time', setupTime, tags);
@@ -129,16 +130,22 @@ async function runOpenFaasUserTransform(
   const metaTags = events[0].metadata ? getMetadata(events[0].metadata) : {};
   const tags = {
     transformerVersionId: userTransformation.versionId,
-    language: userTransformation.language,
     identifier: 'openfaas',
     testMode,
     ...metaTags,
   };
+  const trMetadata = events[0].metadata ? getTransformationMetadata(events[0].metadata) : {};
 
   // check and deploy faas function if not exists
   const functionName = generateFunctionName(userTransformation, libraryVersionIds, testMode);
   if (testMode) {
-    await setOpenFaasUserTransform(userTransformation, libraryVersionIds, functionName, testMode);
+    await setOpenFaasUserTransform(
+      userTransformation,
+      libraryVersionIds,
+      functionName,
+      testMode,
+      trMetadata,
+    );
   }
 
   const invokeTime = new Date();
@@ -156,6 +163,7 @@ async function runOpenFaasUserTransform(
       testMode,
     ),
     testMode,
+    trMetadata,
   );
   stats.timing('run_time', invokeTime, tags);
   return result;

src/util/openfaas/index.js

@@ -23,6 +23,8 @@ const CONFIG_BACKEND_URL = process.env.CONFIG_BACKEND_URL || 'https://api.rudder
 const GEOLOCATION_URL = process.env.GEOLOCATION_URL || '';
 const FAAS_AST_VID = 'ast';
 const FAAS_AST_FN_NAME = 'fn-ast';
+const CUSTOM_NETWORK_POLICY_WORKSPACE_IDS = process.env.CUSTOM_NETWORK_POLICY_WORKSPACE_IDS || '';
+const customNetworkPolicyWorkspaceIds = CUSTOM_NETWORK_POLICY_WORKSPACE_IDS.split(',');
 
 // Initialise node cache
 const functionListCache = new NodeCache();
@@ -111,7 +113,14 @@ const invalidateFnCache = () => {
   functionListCache.set(FUNC_LIST_KEY, []);
 };
 
-const deployFaasFunction = async (functionName, code, versionId, libraryVersionIDs, testMode) => {
+const deployFaasFunction = async (
+  functionName,
+  code,
+  versionId,
+  libraryVersionIDs,
+  testMode,
+  trMetadata = {},
+) => {
   try {
     logger.debug('[Faas] Deploying a faas function');
     let envProcess = 'python index.py';
@@ -132,19 +141,30 @@ const deployFaasFunction = async (functionName, code, versionId, libraryVersionI
     if (GEOLOCATION_URL) {
       envVars.geolocation_url = GEOLOCATION_URL;
     }
+    // labels
+    const labels = {
+      'openfaas-fn': 'true',
+      'parent-component': 'openfaas',
+      'com.openfaas.scale.max': FAAS_MAX_PODS_IN_TEXT,
+      'com.openfaas.scale.min': FAAS_MIN_PODS_IN_TEXT,
+      transformationId: trMetadata.transformationId,
+      workspaceId: trMetadata.workspaceId,
+    };
+    if (
+      trMetadata.workspaceId &&
+      customNetworkPolicyWorkspaceIds.includes(trMetadata.workspaceId)
+    ) {
+      labels['custom-network-policy'] = 'true';
+    }
+
     // TODO: investigate and add more required labels and annotations
     const payload = {
       service: functionName,
       name: functionName,
       image: FAAS_BASE_IMG,
       envProcess,
       envVars,
-      labels: {
-        'openfaas-fn': 'true',
-        'parent-component': 'openfaas',
-        'com.openfaas.scale.max': FAAS_MAX_PODS_IN_TEXT,
-        'com.openfaas.scale.min': FAAS_MIN_PODS_IN_TEXT,
-      },
+      labels,
       annotations: {
         'prometheus.io.scrape': 'true',
       },
@@ -175,14 +195,28 @@ const deployFaasFunction = async (functionName, code, versionId, libraryVersionI
   }
 };
 
-async function setupFaasFunction(functionName, code, versionId, libraryVersionIDs, testMode) {
+async function setupFaasFunction(
+  functionName,
+  code,
+  versionId,
+  libraryVersionIDs,
+  testMode,
+  trMetadata = {},
+) {
   try {
     if (!testMode && isFunctionDeployed(functionName)) {
       logger.debug(`[Faas] Function ${functionName} already deployed`);
       return;
     }
     // deploy faas function
-    await deployFaasFunction(functionName, code, versionId, libraryVersionIDs, testMode);
+    await deployFaasFunction(
+      functionName,
+      code,
+      versionId,
+      libraryVersionIDs,
+      testMode,
+      trMetadata,
+    );
 
     // This api call is only used to check if function is spinned correctly
     await awaitFunctionReadiness(functionName);
@@ -201,6 +235,7 @@ const executeFaasFunction = async (
   versionId,
   libraryVersionIDs,
   testMode,
+  trMetadata = {},
 ) => {
   try {
     logger.debug('[Faas] Invoking faas function');
@@ -217,7 +252,14 @@ const executeFaasFunction = async (
       error.message.includes(`error finding function ${functionName}`)
     ) {
       removeFunctionFromCache(functionName);
-      await setupFaasFunction(functionName, null, versionId, libraryVersionIDs, testMode);
+      await setupFaasFunction(
+        functionName,
+        null,
+        versionId,
+        libraryVersionIDs,
+        testMode,
+        trMetadata,
+      );
       throw new RetryRequestError(`${functionName} not found`);
     }
 

src/v0/destinations/adobe_analytics/transform.js

@@ -11,6 +11,7 @@ const {
   isDefinedAndNotNull,
   isDefinedAndNotNullAndNotEmpty,
   getIntegrationsObj,
+  removeUndefinedAndNullValues,
   simpleProcessRouterDest,
 } = require('../../util');
 const {
@@ -394,7 +395,7 @@ const handleTrack = (message, destinationConfig) => {
       break;
   }
 
-  return payload;
+  return removeUndefinedAndNullValues(payload);
 };
 
 const process = async (event) => {

src/v0/destinations/customerio/transform.js

@@ -12,6 +12,7 @@ const {
   adduserIdFromExternalId,
   getFieldValueFromMessage,
   handleRtTfSingleEventError,
+  validateEventName,
 } = require('../../util');
 
 const logger = require('../../../logger');
@@ -101,6 +102,7 @@ function processSingleMessage(message, destination) {
       break;
     case EventType.TRACK:
       evType = 'event';
+      validateEventName(message.event);
       evName = message.event;
       break;
     case EventType.ALIAS:
@@ -113,6 +115,7 @@ function processSingleMessage(message, destination) {
       logger.error(`could not determine type ${messageType}`);
       throw new InstrumentationError(`could not determine type ${messageType}`);
   }
+  evName = evName ? String(evName) : evName;
   const response = responseBuilder(message, evType, evName, destination, messageType);
 
   // replace default domain with EU data center domainc for EU based account

src/v0/destinations/customerio/util.js

@@ -10,7 +10,6 @@ const {
   defaultDeleteRequestConfig,
   isAppleFamily,
   validateEmail,
-  validateEventType,
 } = require('../../util');
 
 const { EventType, SpecedTraits, TraitsMapping } = require('../../../constants');
@@ -288,7 +287,6 @@ const defaultResponseBuilder = (message, evName, userId, evType, destination, me
       // 100 - len(`Viewed  Screen`) = 86
       trimmedEvName = `Viewed ${truncate(message.event || message.properties.name, 86)} Screen`;
     } else {
-      validateEventType(evName);
       trimmedEvName = truncate(evName, 100);
     }
     // anonymous_id needs to be sent for anon track calls to provide information on which anon user is being tracked

src/v0/destinations/freshmarketer/config.js

@@ -4,23 +4,23 @@ const CONFIG_CATEGORIES = {
   IDENTIFY: {
     name: 'FRESHMARKETERIdentifyConfig',
     type: 'identify',
-    baseUrl: '.myfreshworks.com/crm/sales/api/contacts/upsert',
+    baseUrl: '/crm/sales/api/contacts/upsert',
   },
   GROUP: {
     name: 'FRESHMARKETERGroupConfig',
     type: 'group',
-    baseUrlAccount: '.myfreshworks.com/crm/sales/api/sales_accounts/upsert',
-    baseUrlList: '.myfreshworks.com/crm/sales/api/lists',
+    baseUrlAccount: '/crm/sales/api/sales_accounts/upsert',
+    baseUrlList: '/crm/sales/api/lists',
   },
   SALES_ACTIVITY: {
     name: 'SalesActivityConfig',
-    baseUrlCreate: '.myfreshworks.com/crm/sales/api/sales_activities',
-    baseUrlListAll: '.myfreshworks.com/crm/sales/api/selector/sales_activity_types',
+    baseUrlCreate: '/crm/sales/api/sales_activities',
+    baseUrlListAll: '/crm/sales/api/selector/sales_activity_types',
   },
 };
 
-const DELETE_ENDPOINT = '.myfreshworks.com/crm/sales/api/contacts/';
-const LIFECYCLE_STAGE_ENDPOINT = '.myfreshworks.com/crm/sales/api/selector/lifecycle_stages';
+const DELETE_ENDPOINT = '/crm/sales/api/contacts/';
+const LIFECYCLE_STAGE_ENDPOINT = '/crm/sales/api/selector/lifecycle_stages';
 
 const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname);
 module.exports = {

src/v0/destinations/freshmarketer/utils.js

@@ -203,7 +203,7 @@ const updateAccountWOContact = (payload, Config) => {
  */
 const updateContactWithList = (userId, listId, Config) => {
   const response = defaultRequestConfig();
-  response.endpoint = `https://${Config.domain}.myfreshworks.com/crm/sales/api/lists/${listId}/add_contacts`;
+  response.endpoint = `https://${Config.domain}/crm/sales/api/lists/${listId}/add_contacts`;
   response.headers = getHeaders(Config.apiKey);
   response.body.JSON = {
     ids: [userId],

src/v0/destinations/freshsales/config.js

@@ -4,23 +4,23 @@ const CONFIG_CATEGORIES = {
   IDENTIFY: {
     name: 'identifyConfig',
     type: 'identify',
-    baseUrl: '.myfreshworks.com/crm/sales/api/contacts/upsert',
+    baseUrl: '/crm/sales/api/contacts/upsert',
     method: 'POST',
   },
   GROUP: {
     name: 'groupConfig',
     type: 'group',
-    baseUrlAccount: '.myfreshworks.com/crm/sales/api/sales_accounts/upsert',
+    baseUrlAccount: '/crm/sales/api/sales_accounts/upsert',
     method: 'POST',
   },
   SALES_ACTIVITY: {
     name: 'SalesActivityConfig',
-    baseUrlCreate: '.myfreshworks.com/crm/sales/api/sales_activities',
-    baseUrlListAll: '.myfreshworks.com/crm/sales/api/selector/sales_activity_types',
+    baseUrlCreate: '/crm/sales/api/sales_activities',
+    baseUrlListAll: '/crm/sales/api/selector/sales_activity_types',
   },
 };
 
-const LIFECYCLE_STAGE_ENDPOINT = '.myfreshworks.com/crm/sales/api/selector/lifecycle_stages';
+const LIFECYCLE_STAGE_ENDPOINT = '/crm/sales/api/selector/lifecycle_stages';
 
 const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname);
 module.exports = {