Merge branch 'chore.update-workflow-engine' of github.com:rudderlabs/…

…rudder-transformer into chore.update-workflow-engine
rudderlabs · Sep 14, 2023 · 363678b · 363678b
2 parents 0309abb + ed4ff50
commit 363678b
Show file tree

Hide file tree

Showing 22 changed files with 142 additions and 139 deletions.
diff --git a/src/controllers/misc.ts b/src/controllers/misc.ts
@@ -25,4 +25,22 @@ export default class MiscController {
     ctx.status = 200;
     return ctx;
   }
+
+  public static async getCPUProfile(ctx: Context) {
+    const { seconds } = ctx.query;
+    let secondsData = 10;
+    // if seconds is not null and is not array then parseInt
+    if (seconds && !Array.isArray(seconds)) {
+      secondsData = parseInt(seconds, 10);
+    }
+    ctx.body = await MiscService.getCPUProfile(secondsData);
+    ctx.status = 200;
+    return ctx;
+  }
+
+  public static async getHeapProfile(ctx: Context) {
+    ctx.body = await MiscService.getHeapProfile();
+    ctx.status = 200;
+    return ctx;
+  }
 }
diff --git a/src/index.ts b/src/index.ts
@@ -7,17 +7,19 @@ import cluster from './util/cluster';
 import { router } from './legacy/router';
 import { testRouter } from './testRouter';
 import { metricsRouter } from './routes/metricsRouter';
-import { addStatMiddleware, addRequestSizeMiddleware, addPyroscopeMiddleware } from './middleware';
+import { addStatMiddleware, addRequestSizeMiddleware, initPyroscope } from './middleware';
 import { logProcessInfo } from './util/utils';
 import { applicationRoutes, addSwaggerRoutes } from './routes';
 import { RedisDB } from './util/redis/redisConnector';
 
 dotenv.config();
 const clusterEnabled = process.env.CLUSTER_ENABLED !== 'false';
 const useUpdatedRoutes = process.env.ENABLE_NEW_ROUTES !== 'false';
-const port = parseInt(process.env.PORT || '9090', 10);
+const port = parseInt(process.env.PORT ?? '9090', 10);
 const metricsPort = parseInt(process.env.METRICS_PORT || '9091', 10);
 
+initPyroscope();
+
 const app = new Koa();
 addStatMiddleware(app);
 
@@ -30,7 +32,6 @@ app.use(
     jsonLimit: '200mb',
   }),
 );
-addPyroscopeMiddleware(app);
 addRequestSizeMiddleware(app);
 addSwaggerRoutes(app);
 

diff --git a/src/middleware.js b/src/middleware.js
@@ -1,48 +1,19 @@
 const Pyroscope = require('@pyroscope/nodejs');
 const stats = require('./util/stats');
-const logger = require('./logger');
 
-Pyroscope.init({
-  appName: 'rudder-transformer',
-});
-
-async function handlerCpu(ctx) {
-  try {
-    const p = await Pyroscope.collectCpu(Number(ctx.query.seconds));
-    ctx.body = p;
-    ctx.status = 200;
-  } catch (e) {
-    logger.error(e);
-    ctx.status = 500;
-  }
-}
-
-async function handlerHeap(ctx) {
-  try {
-    const p = await Pyroscope.collectHeap();
-    ctx.body = p;
-    ctx.status = 200;
-  } catch (e) {
-    logger.error(e);
-    ctx.status = 500;
-  }
+function initPyroscope() {
+  Pyroscope.init({
+    appName: 'rudder-transformer',
+  });
+  Pyroscope.startHeapCollecting();
 }
 
-function pyroscopeMiddleware() {
-  Pyroscope.startHeapCollecting();
-  return (ctx, next) => {
-    if (ctx.method === 'GET' && ctx.path === '/debug/pprof/profile') {
-      return handlerCpu(ctx).then(() => next());
-    }
-    if (ctx.method === 'GET' && ctx.path === '/debug/pprof/heap') {
-      return handlerHeap(ctx).then(() => next());
-    }
-    return next();
-  };
+function getCPUProfile(seconds) {
+  return Pyroscope.collectCpu(seconds);
 }
 
-function addPyroscopeMiddleware(app) {
-  app.use(pyroscopeMiddleware());
+function getHeapProfile() {
+  return Pyroscope.collectHeap();
 }
 
 function durationMiddleware() {
@@ -90,5 +61,7 @@ function addRequestSizeMiddleware(app) {
 module.exports = {
   addStatMiddleware,
   addRequestSizeMiddleware,
-  addPyroscopeMiddleware,
+  getHeapProfile,
+  getCPUProfile,
+  initPyroscope,
 };
diff --git a/src/routes/misc.ts b/src/routes/misc.ts
@@ -10,5 +10,7 @@ router.get('/transformerBuildVersion', MiscController.buildVersion); // depricia
 router.get('/buildVersion', MiscController.buildVersion);
 router.get('/version', MiscController.version);
 router.get('/features', MiscController.features);
+router.get('/debug/pprof/profile', MiscController.getCPUProfile);
+router.get('/debug/pprof/heap', MiscController.getHeapProfile);
 
 export const miscRoutes = router.routes();
diff --git a/src/services/misc.ts b/src/services/misc.ts
@@ -3,6 +3,7 @@ import path from 'path';
 import { Context } from 'koa';
 import { DestHandlerMap } from '../constants/destinationCanonicalNames';
 import { Metadata } from '../types';
+import { getCPUProfile, getHeapProfile, } from '../middleware';
 
 export default class MiscService {
   public static getDestHandler(dest: string, version: string) {
@@ -32,7 +33,7 @@ export default class MiscService {
 
   public static getMetaTags(metadata: Metadata) {
     if (!metadata) {
-      return {}
+      return {};
     }
     return {
       sourceType: metadata.sourceType,
@@ -62,4 +63,12 @@ export default class MiscService {
     const obj = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../features.json'), 'utf8'));
     return JSON.stringify(obj);
   }
+
+  public static async getCPUProfile(seconds: number) {
+    return getCPUProfile(seconds);
+  }
+
+  public static async getHeapProfile() {
+    return getHeapProfile()
+  }
 }
diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/config.js b/src/v0/destinations/google_adwords_enhanced_conversions/config.js
@@ -1,6 +1,6 @@
 const { getMappingConfig } = require('../../util');
 
-const BASE_ENDPOINT = 'https://googleads.googleapis.com/v13/customers';
+const BASE_ENDPOINT = 'https://googleads.googleapis.com/v14/customers';
 
 const CONFIG_CATEGORIES = {
   TRACK_CONFIG: { type: 'track', name: 'trackConfig' },

diff --git a/src/v0/destinations/google_adwords_offline_conversions/config.js b/src/v0/destinations/google_adwords_offline_conversions/config.js
@@ -1,6 +1,6 @@
 const { getMappingConfig } = require('../../util');
 
-const API_VERSION = 'v13';
+const API_VERSION = 'v14';
 
 const BASE_ENDPOINT = `https://googleads.googleapis.com/${API_VERSION}/customers/:customerId`;
 

diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js
@@ -161,7 +161,7 @@ describe('getExisitingUserIdentifier util tests', () => {
 describe('getClickConversionPayloadAndEndpoint util tests', () => {
   it('getClickConversionPayloadAndEndpoint flow check when default field identifier is present', () => {
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v13/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {
@@ -187,7 +187,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
     delete fittingPayload.traits.email;
     delete fittingPayload.properties.email;
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v13/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {
@@ -215,7 +215,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
     delete fittingPayload.traits.phone;
     delete fittingPayload.properties.email;
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v13/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {
@@ -251,7 +251,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => {
       },
     ];
     let expectedOutput = {
-      endpoint: 'https://googleads.googleapis.com/v13/customers/9625812972:uploadClickConversions',
+      endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions',
       payload: {
         conversions: [
           {

diff --git a/src/v0/destinations/google_adwords_remarketing_lists/config.js b/src/v0/destinations/google_adwords_remarketing_lists/config.js
@@ -1,6 +1,6 @@
 const { getMappingConfig } = require('../../util');
 
-const BASE_ENDPOINT = 'https://googleads.googleapis.com/v13/customers';
+const BASE_ENDPOINT = 'https://googleads.googleapis.com/v14/customers';
 const CONFIG_CATEGORIES = {
   AUDIENCE_LIST: { type: 'audienceList', name: 'offlineDataJobs' },
   ADDRESSINFO: { type: 'addressInfo', name: 'addressInfo' },

diff --git a/test/__mocks__/data/google_adwords_offline_conversion/response.json b/test/__mocks__/data/google_adwords_offline_conversion/response.json
@@ -1,17 +1,17 @@
 {
-  "https://googleads.googleapis.com/v13/customers/11122233331/offlineUserDataJobs:create": {
+  "https://googleads.googleapis.com/v14/customers/11122233331/offlineUserDataJobs:create": {
     "data": {
       "resourceName": "customers/111-222-3333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_FOR_ADD_FAILURE"
     },
     "status": 200
   },
-  "https://googleads.googleapis.com/v13/customers/1112223333/offlineUserDataJobs:create": {
+  "https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs:create": {
     "data": {
       "resourceName": "customers/111-222-3333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID"
     },
     "status": 200
   },
-  "https://googleads.googleapis.com/v13/customers/customerid/offlineUserDataJobs:create": {
+  "https://googleads.googleapis.com/v14/customers/customerid/offlineUserDataJobs:create": {
     "status": 401,
     "data": {
       "error": {
@@ -21,11 +21,11 @@
       }
     }
   },
-  "https://googleads.googleapis.com/v13/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:addOperations": {
+  "https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:addOperations": {
     "status": 200,
     "data": {}
   },
-  "https://googleads.googleapis.com/v13/customers/11122233331/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_FOR_ADD_FAILURE:addOperations": {
+  "https://googleads.googleapis.com/v14/customers/11122233331/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_FOR_ADD_FAILURE:addOperations": {
     "status": 400,
     "data": {
       "error": {
@@ -34,7 +34,7 @@
         "status": "INVALID_ARGUMENT",
         "details": [
           {
-            "@type": "type.googleapis.com/google.ads.googleads.v13.errors.GoogleAdsFailure",
+            "@type": "type.googleapis.com/google.ads.googleads.v14.errors.GoogleAdsFailure",
             "errors": [
               {
                 "errorCode": {
@@ -67,13 +67,13 @@
       }
     }
   },
-  "https://googleads.googleapis.com/v13/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:run": {
+  "https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:run": {
     "status": 200,
     "data": {
       "name": "customers/111-222-3333/operations/abcd="
     }
   },
-  "https://googleads.googleapis.com/v13/customers/customerid/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_ADD_FAILURE:addOperations": {
+  "https://googleads.googleapis.com/v14/customers/customerid/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_ADD_FAILURE:addOperations": {
     "status": 400,
     "data": {
       "error": {
@@ -82,7 +82,7 @@
         "status": "INVALID_ARGUMENT",
         "details": [
           {
-            "@type": "type.googleapis.com/google.ads.googleads.v13.errors.GoogleAdsFailure",
+            "@type": "type.googleapis.com/google.ads.googleads.v14.errors.GoogleAdsFailure",
             "errors": [
               {
                 "errorCode": {

diff --git a/test/__mocks__/data/google_adwords_remarketing_lists/proxy_response.json b/test/__mocks__/data/google_adwords_remarketing_lists/proxy_response.json
@@ -1,23 +1,23 @@
 {
-  "https://googleads.googleapis.com/v13/customers/7693729833/offlineUserDataJobs:create": {
+  "https://googleads.googleapis.com/v14/customers/7693729833/offlineUserDataJobs:create": {
     "status": 200,
     "data": {
       "resourceName": "customers/9249589672/offlineUserDataJobs/18025019461"
     }
   },
-  "https://googleads.googleapis.com/v13/customers/7693729833/offlineUserDataJobs/18025019461:addOperations": {
+  "https://googleads.googleapis.com/v14/customers/7693729833/offlineUserDataJobs/18025019461:addOperations": {
     "status": 200
   },
-  "https://googleads.googleapis.com/v13/customers/7693729833/offlineUserDataJobs/18025019461:run": {
+  "https://googleads.googleapis.com/v14/customers/7693729833/offlineUserDataJobs/18025019461:run": {
     "status": 200
   },
-  "https://googleads.googleapis.com/v13/customers/7693729834/offlineUserDataJobs:create": {
+  "https://googleads.googleapis.com/v14/customers/7693729834/offlineUserDataJobs:create": {
     "status": 200,
     "data": {
       "resourceName": "customers/9249589672/offlineUserDataJobs/18025019462"
     }
   },
-  "https://googleads.googleapis.com/v13/customers/7693729834/offlineUserDataJobs/18025019462:addOperations": {
+  "https://googleads.googleapis.com/v14/customers/7693729834/offlineUserDataJobs/18025019462:addOperations": {
     "response": {
       "data": {
         "error": {
@@ -60,7 +60,7 @@
       "status": 400
     }
   },
-  "https://googleads.googleapis.com/v13/customers/1234567890/googleAds:searchStream": {
+  "https://googleads.googleapis.com/v14/customers/1234567890/googleAds:searchStream": {
     "response": {
       "data": [
         {
@@ -74,11 +74,11 @@
       "status": 401
     }
   },
-  "https://googleads.googleapis.com/v13/customers/1234567899/googleAds:searchStream": {
+  "https://googleads.googleapis.com/v14/customers/1234567899/googleAds:searchStream": {
     "response": "",
     "code": "ECONNREFUSED"
   },
-  "https://googleads.googleapis.com/v13/customers/1234567891/googleAds:searchStream": {
+  "https://googleads.googleapis.com/v14/customers/1234567891/googleAds:searchStream": {
     "data": [
       {
         "results": [
@@ -95,7 +95,7 @@
     ],
     "status": 200
   },
-  "https://googleads.googleapis.com/v13/customers/1234567891:uploadConversionAdjustments": {
+  "https://googleads.googleapis.com/v14/customers/1234567891:uploadConversionAdjustments": {
     "data": [
       {
         "adjustmentType": "ENHANCEMENT",

diff --git a/test/__tests__/data/google_adwords_enhanced_conversions_output.json b/test/__tests__/data/google_adwords_enhanced_conversions_output.json
@@ -3,7 +3,7 @@
     "version": "1",
     "type": "REST",
     "method": "POST",
-    "endpoint": "https://googleads.googleapis.com/v13/customers/1234567890:uploadConversionAdjustments",
+    "endpoint": "https://googleads.googleapis.com/v14/customers/1234567890:uploadConversionAdjustments",
     "headers": {
       "Authorization": "Bearer abcd1234",
       "Content-Type": "application/json",
@@ -91,7 +91,7 @@
     "version": "1",
     "type": "REST",
     "method": "POST",
-    "endpoint": "https://googleads.googleapis.com/v13/customers/1234567890:uploadConversionAdjustments",
+    "endpoint": "https://googleads.googleapis.com/v14/customers/1234567890:uploadConversionAdjustments",
     "headers": {
       "Authorization": "Bearer abcd1234",
       "Content-Type": "application/json",
@@ -204,7 +204,7 @@
     "version": "1",
     "type": "REST",
     "method": "POST",
-    "endpoint": "https://googleads.googleapis.com/v13/customers/1234567890:uploadConversionAdjustments",
+    "endpoint": "https://googleads.googleapis.com/v14/customers/1234567890:uploadConversionAdjustments",
     "headers": {
       "Authorization": "Bearer abcd1234",
       "Content-Type": "application/json",
@@ -260,7 +260,7 @@
     "version": "1",
     "type": "REST",
     "method": "POST",
-    "endpoint": "https://googleads.googleapis.com/v13/customers/1234567890:uploadConversionAdjustments",
+    "endpoint": "https://googleads.googleapis.com/v14/customers/1234567890:uploadConversionAdjustments",
     "headers": {
       "Authorization": "Bearer abcd1234",
       "Content-Type": "application/json",