Merge branch 'develop' into IDP-9-run-actions-on-self-hosted-transfor…

…mers
rudderlabs · Sep 14, 2023 · ab1c118 · ab1c118
2 parents 787bfcc + 7d268f0
commit ab1c118
Show file tree

Hide file tree

Showing 26 changed files with 223 additions and 147 deletions.
diff --git a/src/controllers/misc.ts b/src/controllers/misc.ts
@@ -25,4 +25,22 @@ export default class MiscController {
     ctx.status = 200;
     return ctx;
   }
+
+  public static async getCPUProfile(ctx: Context) {
+    const { seconds } = ctx.query;
+    let secondsData = 10;
+    // if seconds is not null and is not array then parseInt
+    if (seconds && !Array.isArray(seconds)) {
+      secondsData = parseInt(seconds, 10);
+    }
+    ctx.body = await MiscService.getCPUProfile(secondsData);
+    ctx.status = 200;
+    return ctx;
+  }
+
+  public static async getHeapProfile(ctx: Context) {
+    ctx.body = await MiscService.getHeapProfile();
+    ctx.status = 200;
+    return ctx;
+  }
 }
diff --git a/src/index.ts b/src/index.ts
@@ -7,17 +7,19 @@ import cluster from './util/cluster';
 import { router } from './legacy/router';
 import { testRouter } from './testRouter';
 import { metricsRouter } from './routes/metricsRouter';
-import { addStatMiddleware, addRequestSizeMiddleware, addPyroscopeMiddleware } from './middleware';
+import { addStatMiddleware, addRequestSizeMiddleware, initPyroscope } from './middleware';
 import { logProcessInfo } from './util/utils';
 import { applicationRoutes, addSwaggerRoutes } from './routes';
 import { RedisDB } from './util/redis/redisConnector';
 
 dotenv.config();
 const clusterEnabled = process.env.CLUSTER_ENABLED !== 'false';
 const useUpdatedRoutes = process.env.ENABLE_NEW_ROUTES !== 'false';
-const port = parseInt(process.env.PORT || '9090', 10);
+const port = parseInt(process.env.PORT ?? '9090', 10);
 const metricsPort = parseInt(process.env.METRICS_PORT || '9091', 10);
 
+initPyroscope();
+
 const app = new Koa();
 addStatMiddleware(app);
 
@@ -30,7 +32,6 @@ app.use(
     jsonLimit: '200mb',
   }),
 );
-addPyroscopeMiddleware(app);
 addRequestSizeMiddleware(app);
 addSwaggerRoutes(app);
 

diff --git a/src/middleware.js b/src/middleware.js
@@ -1,48 +1,19 @@
 const Pyroscope = require('@pyroscope/nodejs');
 const stats = require('./util/stats');
-const logger = require('./logger');
 
-Pyroscope.init({
-  appName: 'rudder-transformer',
-});
-
-async function handlerCpu(ctx) {
-  try {
-    const p = await Pyroscope.collectCpu(Number(ctx.query.seconds));
-    ctx.body = p;
-    ctx.status = 200;
-  } catch (e) {
-    logger.error(e);
-    ctx.status = 500;
-  }
-}
-
-async function handlerHeap(ctx) {
-  try {
-    const p = await Pyroscope.collectHeap();
-    ctx.body = p;
-    ctx.status = 200;
-  } catch (e) {
-    logger.error(e);
-    ctx.status = 500;
-  }
+function initPyroscope() {
+  Pyroscope.init({
+    appName: 'rudder-transformer',
+  });
+  Pyroscope.startHeapCollecting();
 }
 
-function pyroscopeMiddleware() {
-  Pyroscope.startHeapCollecting();
-  return (ctx, next) => {
-    if (ctx.method === 'GET' && ctx.path === '/debug/pprof/profile') {
-      return handlerCpu(ctx).then(() => next());
-    }
-    if (ctx.method === 'GET' && ctx.path === '/debug/pprof/heap') {
-      return handlerHeap(ctx).then(() => next());
-    }
-    return next();
-  };
+function getCPUProfile(seconds) {
+  return Pyroscope.collectCpu(seconds);
 }
 
-function addPyroscopeMiddleware(app) {
-  app.use(pyroscopeMiddleware());
+function getHeapProfile() {
+  return Pyroscope.collectHeap();
 }
 
 function durationMiddleware() {
@@ -90,5 +61,7 @@ function addRequestSizeMiddleware(app) {
 module.exports = {
   addStatMiddleware,
   addRequestSizeMiddleware,
-  addPyroscopeMiddleware,
+  getHeapProfile,
+  getCPUProfile,
+  initPyroscope,
 };
diff --git a/src/routes/misc.ts b/src/routes/misc.ts
@@ -10,5 +10,7 @@ router.get('/transformerBuildVersion', MiscController.buildVersion); // depricia
 router.get('/buildVersion', MiscController.buildVersion);
 router.get('/version', MiscController.version);
 router.get('/features', MiscController.features);
+router.get('/debug/pprof/profile', MiscController.getCPUProfile);
+router.get('/debug/pprof/heap', MiscController.getHeapProfile);
 
 export const miscRoutes = router.routes();
diff --git a/src/services/misc.ts b/src/services/misc.ts
@@ -3,6 +3,7 @@ import path from 'path';
 import { Context } from 'koa';
 import { DestHandlerMap } from '../constants/destinationCanonicalNames';
 import { Metadata } from '../types';
+import { getCPUProfile, getHeapProfile, } from '../middleware';
 
 export default class MiscService {
   public static getDestHandler(dest: string, version: string) {
@@ -32,7 +33,7 @@ export default class MiscService {
 
   public static getMetaTags(metadata: Metadata) {
     if (!metadata) {
-      return {}
+      return {};
     }
     return {
       sourceType: metadata.sourceType,
@@ -62,4 +63,12 @@ export default class MiscService {
     const obj = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../features.json'), 'utf8'));
     return JSON.stringify(obj);
   }
+
+  public static async getCPUProfile(seconds: number) {
+    return getCPUProfile(seconds);
+  }
+
+  public static async getHeapProfile() {
+    return getHeapProfile()
+  }
 }
diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/config.js b/src/v0/destinations/google_adwords_enhanced_conversions/config.js
@@ -1,6 +1,6 @@
 const { getMappingConfig } = require('../../util');
 
-const BASE_ENDPOINT = 'https://googleads.googleapis.com/v13/customers';
+const BASE_ENDPOINT = 'https://googleads.googleapis.com/v14/customers';
 
 const CONFIG_CATEGORIES = {
   TRACK_CONFIG: { type: 'track', name: 'trackConfig' },

diff --git a/src/v0/destinations/google_adwords_offline_conversions/config.js b/src/v0/destinations/google_adwords_offline_conversions/config.js
@@ -1,6 +1,6 @@
 const { getMappingConfig } = require('../../util');
 
-const API_VERSION = 'v13';
+const API_VERSION = 'v14';
 
 const BASE_ENDPOINT = `https://googleads.googleapis.com/${API_VERSION}/customers/:customerId`;
 

diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js
@@ -161,7 +161,7 @@ describe('getExisitingUserIdentifier util tests', () => {
 describe('getClickConversionPayloadAndEndpoint util tests', () => {
   it('getClickConversionPayloadAndEndpoint flow check when default field identifier is present', () => {
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v13/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {
@@ -187,7 +187,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
     delete fittingPayload.traits.email;
     delete fittingPayload.properties.email;
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v13/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {
@@ -215,7 +215,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
     delete fittingPayload.traits.phone;
     delete fittingPayload.properties.email;
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v13/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {
@@ -251,7 +251,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
       },
     ];
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v13/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {

diff --git a/src/v0/destinations/google_adwords_remarketing_lists/config.js b/src/v0/destinations/google_adwords_remarketing_lists/config.js
@@ -1,6 +1,6 @@
 const { getMappingConfig } = require('../../util');
 
-const BASE_ENDPOINT = 'https://googleads.googleapis.com/v13/customers';
+const BASE_ENDPOINT = 'https://googleads.googleapis.com/v14/customers';
 const CONFIG_CATEGORIES = {
   AUDIENCE_LIST: { type: 'audienceList', name: 'offlineDataJobs' },
   ADDRESSINFO: { type: 'addressInfo', name: 'addressInfo' },

diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js
@@ -130,6 +130,37 @@ const processRevenueEvents = (message, destination, revenueValue) => {
   return responseBuilderSimple(payload, message, 'revenue', destination.Config);
 };
 
+/**
+ * This function is used to process the incremental properties
+ * ref :- https://developer.mixpanel.com/reference/profile-numerical-add
+ * @param {*} message
+ * @param {*} destination
+ * @param {*} propIncrements
+ * @returns
+ */
+const processIncrementalProperties = (message, destination, propIncrements) => {
+  const payload = {
+    $add: {},
+    $token: destination.Config.token,
+    $distinct_id: message.userId || message.anonymousId,
+  };
+
+  if (destination?.Config.identityMergeApi === 'simplified') {
+    payload.$distinct_id = message.userId || `$device:${message.anonymousId}`;
+  }
+
+  Object.keys(message.properties).forEach((prop) => {
+    const value = message.properties[prop];
+    if (value && propIncrements.includes(prop)) {
+      payload.$add[prop] = value;
+    }
+  });
+
+  return Object.keys(payload.$add).length > 0
+    ? responseBuilderSimple(payload, message, 'incremental_properties', destination.Config)
+    : null;
+};
+
 const getEventValueForTrackEvent = (message, destination) => {
   const mappedProperties = constructPayload(message, mPEventPropertiesConfigJson);
   // This is to conform with SDKs sending timestamp component with messageId
@@ -178,6 +209,14 @@ const processTrack = (message, destination) => {
   if (revenue) {
     returnValue.push(processRevenueEvents(message, destination, revenue));
   }
+
+  if (Array.isArray(destination.Config.propIncrements)) {
+    const propIncrements = destination.Config.propIncrements.map((item) => item.property);
+    const response = processIncrementalProperties(message, destination, propIncrements);
+    if (response) {
+      returnValue.push(response);
+    }
+  }
   returnValue.push(getEventValueForTrackEvent(message, destination));
   return returnValue;
 };

diff --git a/src/v0/destinations/mp/util.test.js b/src/v0/destinations/mp/util.test.js
@@ -1,7 +1,4 @@
-const {
-  combineBatchRequestsWithSameJobIds,
-  combineBatchRequestsWithSameJobIds2,
-} = require('./util');
+const { combineBatchRequestsWithSameJobIds } = require('./util');
 
 const destinationMock = {
   Config: {

diff --git a/test/__mocks__/data/google_adwords_offline_conversion/response.json b/test/__mocks__/data/google_adwords_offline_conversion/response.json
@@ -1,17 +1,17 @@
 {
-  "https://googleads.googleapis.com/v13/customers/11122233331/offlineUserDataJobs:create": {
+  "https://googleads.googleapis.com/v14/customers/11122233331/offlineUserDataJobs:create": {
     "data": {
       "resourceName": "customers/111-222-3333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_FOR_ADD_FAILURE"
     },
     "status": 200
   },
-  "https://googleads.googleapis.com/v13/customers/1112223333/offlineUserDataJobs:create": {
+  "https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs:create": {
     "data": {
       "resourceName": "customers/111-222-3333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID"
     },
     "status": 200
   },
-  "https://googleads.googleapis.com/v13/customers/customerid/offlineUserDataJobs:create": {
+  "https://googleads.googleapis.com/v14/customers/customerid/offlineUserDataJobs:create": {
     "status": 401,
     "data": {
       "error": {
@@ -21,11 +21,11 @@
       }
     }
   },
-  "https://googleads.googleapis.com/v13/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:addOperations": {
+  "https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:addOperations": {
     "status": 200,
     "data": {}
   },
-  "https://googleads.googleapis.com/v13/customers/11122233331/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_FOR_ADD_FAILURE:addOperations": {
+  "https://googleads.googleapis.com/v14/customers/11122233331/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_FOR_ADD_FAILURE:addOperations": {
     "status": 400,
     "data": {
       "error": {
@@ -34,7 +34,7 @@
         "status": "INVALID_ARGUMENT",
         "details": [
           {
-            "@type": "type.googleapis.com/google.ads.googleads.v13.errors.GoogleAdsFailure",
+            "@type": "type.googleapis.com/google.ads.googleads.v14.errors.GoogleAdsFailure",
             "errors": [
               {
                 "errorCode": {
@@ -67,13 +67,13 @@
       }
     }
   },
-  "https://googleads.googleapis.com/v13/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:run": {
+  "https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:run": {
     "status": 200,
     "data": {
       "name": "customers/111-222-3333/operations/abcd="
     }
   },
-  "https://googleads.googleapis.com/v13/customers/customerid/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_ADD_FAILURE:addOperations": {
+  "https://googleads.googleapis.com/v14/customers/customerid/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_ADD_FAILURE:addOperations": {
     "status": 400,
     "data": {
       "error": {
@@ -82,7 +82,7 @@
         "status": "INVALID_ARGUMENT",
         "details": [
           {
-            "@type": "type.googleapis.com/google.ads.googleads.v13.errors.GoogleAdsFailure",
+            "@type": "type.googleapis.com/google.ads.googleads.v14.errors.GoogleAdsFailure",
             "errors": [
               {
                 "errorCode": {