chore: add logger at integration level (#3401)

benchmark/metaLogger.js

@@ -2,30 +2,30 @@
 
 const logger = require('../src/logger');
 
-logger.setLogLevel(Number.POSITIVE_INFINITY);
+logger.setLogLevel('random');
 
 const debug = (...args) => {
-  logger.setLogLevel(logger.levelDebug);
+  logger.setLogLevel('debug');
   logger.debug(...args);
-  logger.setLogLevel(Number.POSITIVE_INFINITY);
+  logger.setLogLevel('random');
 };
 
 const info = (...args) => {
-  logger.setLogLevel(logger.levelInfo);
+  logger.setLogLevel('info');
   logger.info(...args);
-  logger.setLogLevel(Number.POSITIVE_INFINITY);
+  logger.setLogLevel('random');
 };
 
 const warn = (...args) => {
-  logger.setLogLevel(logger.levelWarn);
+  logger.setLogLevel('warn');
   logger.warn(...args);
-  logger.setLogLevel(Number.POSITIVE_INFINITY);
+  logger.setLogLevel('random');
 };
 
 const error = (...args) => {
-  logger.setLogLevel(logger.levelError);
+  logger.setLogLevel('error');
   logger.error(...args);
-  logger.setLogLevel(Number.POSITIVE_INFINITY);
+  logger.setLogLevel('random');
 };
 
 module.exports = {

package.json

@@ -64,7 +64,7 @@
     "@koa/router": "^12.0.0",
     "@ndhoule/extend": "^2.0.0",
     "@pyroscope/nodejs": "^0.2.9",
-    "@rudderstack/integrations-lib": "^0.2.8",
+    "@rudderstack/integrations-lib": "^0.2.10",
     "@rudderstack/json-template-engine": "^0.13.2",
     "@rudderstack/workflow-engine": "^0.8.2",
     "@shopify/jest-koa-mocks": "^5.1.1",

src/adapters/network.js

@@ -45,21 +45,28 @@ const networkClientConfigs = {
   httpsAgent: new https.Agent({ keepAlive: true }),
 };
 
-const fireHTTPStats = (clientResponse, startTime, statTags) => {
-  const destType = statTags.destType ? statTags.destType : '';
-  const feature = statTags.feature ? statTags.feature : '';
-  const endpointPath = statTags.endpointPath ? statTags.endpointPath : '';
-  const requestMethod = statTags.requestMethod ? statTags.requestMethod : '';
-  const module = statTags.module ? statTags.module : '';
-  const statusCode = clientResponse.success ? clientResponse.response.status : '';
+const fireOutgoingReqStats = ({
+  destType,
+  feature,
+  endpointPath,
+  requestMethod,
+  module,
+  metadata = {},
+  startTime,
+  statusCode,
+  clientResponse,
+}) => {
+  const logMetaInfo = log.getLogMetadata(metadata);
   stats.timing('outgoing_request_latency', startTime, {
+    ...logMetaInfo,
     feature,
     destType,
     endpointPath,
     requestMethod,
     module,
   });
   stats.counter('outgoing_request_count', 1, {
+    ...logMetaInfo,
     feature,
     destType,
     endpointPath,
@@ -70,6 +77,36 @@ const fireHTTPStats = (clientResponse, startTime, statTags) => {
   });
 };
 
+const fireHTTPStats = (clientResponse, startTime, statTags) => {
+  const destType = statTags.destType ? statTags.destType : '';
+  const feature = statTags.feature ? statTags.feature : '';
+  const endpointPath = statTags.endpointPath ? statTags.endpointPath : '';
+  const requestMethod = statTags.requestMethod ? statTags.requestMethod : '';
+  const module = statTags.module ? statTags.module : '';
+  const statusCode = clientResponse.success ? clientResponse.response.status : '';
+  const defArgs = {
+    destType,
+    endpointPath,
+    feature,
+    module,
+    requestMethod,
+    statusCode,
+    startTime,
+    clientResponse,
+  };
+  if (statTags?.metadata) {
+    const metadata = !Array.isArray(statTags?.metadata) ? [statTags.metadata] : statTags.metadata;
+    metadata?.forEach((m) => {
+      fireOutgoingReqStats({
+        ...defArgs,
+        metadata: m,
+      });
+    });
+    return;
+  }
+  fireOutgoingReqStats(defArgs);
+};
+
 const enhanceRequestOptions = (options) => {
   const requestOptions = {
     ...networkClientConfigs,
@@ -322,25 +359,6 @@ const prepareProxyRequest = (request) => {
   return removeUndefinedValues({ endpoint, data, params, headers, method, config });
 };
 
-/**
- * depricating: handles proxying requests to destinations from server, expects requsts in "defaultRequestConfig"
- * note: needed for test api
- * @param {*} request
- * @returns
- */
-const proxyRequest = async (request, destType) => {
-  const { endpoint, data, method, params, headers } = prepareProxyRequest(request);
-  const requestOptions = {
-    url: endpoint,
-    data,
-    params,
-    headers,
-    method,
-  };
-  const response = await httpSend(requestOptions, { feature: 'proxy', destType });
-  return response;
-};
-
 /**
  * handles http request and sends the response in a simple format that is followed in transformer
  *
@@ -392,6 +410,38 @@ const handleHttpRequest = async (requestType = 'post', ...httpArgs) => {
   return { httpResponse, processedResponse };
 };
 
+/**
+ * depricating: handles proxying requests to destinations from server, expects requsts in "defaultRequestConfig"
+ * note: needed for test api
+ * @param {*} request
+ * @returns
+ */
+const proxyRequest = async (request, destType) => {
+  const { metadata } = request;
+  const { endpoint, data, method, params, headers } = prepareProxyRequest(request);
+  const requestOptions = {
+    url: endpoint,
+    data,
+    params,
+    headers,
+    method,
+  };
+  log.requestLog(`[${destType.toUpperCase()}] delivering data`, {
+    metadata,
+    requestDetails: {
+      body: data,
+      url: endpoint,
+      method,
+    },
+  });
+  const response = await httpSend(requestOptions, {
+    feature: 'proxy',
+    destType,
+    metadata,
+  });
+  return response;
+};
+
 module.exports = {
   httpSend,
   httpGET,

src/adapters/utils/networkUtils.js

@@ -144,10 +144,11 @@ const processAxiosResponse = (clientResponse) => {
     }
     // non 2xx status handling for axios response
     if (response) {
-      const { data, status } = response;
+      const { data, status, headers } = response;
       return {
         response: data || '',
         status: status || 500,
+        ...(isDefinedAndNotNullAndNotEmpty(headers) ? { headers } : {}),
       };
     }
     // (edge case) response and code is not present
@@ -157,10 +158,11 @@ const processAxiosResponse = (clientResponse) => {
     };
   }
   // success(2xx) axios response
-  const { data, status } = clientResponse.response;
+  const { data, status, headers } = clientResponse.response;
   return {
     response: data || '',
     status: status || 500,
+    ...(isDefinedAndNotNullAndNotEmpty(headers) ? { headers } : {}),
   };
 };
 

src/cdk/v2/destinations/gladly/procWorkflow.yaml

@@ -61,7 +61,8 @@ steps:
         headers: $.getHeaders(.destination)
       }
       const endpoint = $.getEndpoint(.destination) + "?" + $.getQueryParams($.context.payload);
-      const rawResponse = await $.httpGET(endpoint,requestOptions)
+      const reqStats = {metadata:.metadata, module: 'router',feature: "transformation", destType:"gladly",requestMethod:"get",endpointPath:"/api/v1/customer-profiles"}
+      const rawResponse = await $.httpGET(endpoint,requestOptions, reqStats)
       const processedResponse = $.processAxiosResponse(rawResponse)
       processedResponse
 

src/cdk/v2/handler.ts

@@ -17,6 +17,8 @@ import {
   isCdkV2Destination,
 } from './utils';
 
+import logger from '../../logger';
+
 const defTags = {
   [tags.TAG_NAMES.IMPLEMENTATION]: tags.IMPLEMENTATIONS.CDK_V2,
 };
@@ -82,12 +84,12 @@ export async function processCdkV2Workflow(
   destType: string,
   parsedEvent: FixMe,
   feature: string,
-  logger: FixMe,
   requestMetadata: NonNullable<unknown> = {},
   bindings: Record<string, FixMe> = {},
 ) {
   try {
-    logger.debug(`Processing cdkV2 workflow`);
+    logger.debug(`Processing cdkV2 workflow`, { destType });
+
     const workflowEngine = await getCachedWorkflowEngine(destType, feature, bindings);
     return await executeWorkflow(workflowEngine, parsedEvent, requestMetadata);
   } catch (error) {

src/controllers/bulkUpload.ts

@@ -1,12 +1,12 @@
 /* eslint-disable global-require, import/no-dynamic-require, @typescript-eslint/no-unused-vars */
-import { structuredLogger as logger } from '@rudderstack/integrations-lib';
 import { client as errNotificationClient } from '../util/errorNotifier';
 import {
   getDestFileUploadHandler,
   getJobStatusHandler,
   getPollStatusHandler,
 } from '../util/fetchDestinationHandlers';
 import { CatchErr, ContextBodySimple } from '../util/types';
+import logger from '../logger';
 // TODO: To be refactored and redisgned
 
 const ERROR_MESSAGE_PROCESSOR_STRING = 'Error occurred while processing payload.';

src/controllers/delivery.ts

@@ -1,9 +1,6 @@
 /* eslint-disable prefer-destructuring */
 /* eslint-disable sonarjs/no-duplicate-string */
-import {
-  isDefinedAndNotNullAndNotEmpty,
-  structuredLogger as logger,
-} from '@rudderstack/integrations-lib';
+import { isDefinedAndNotNullAndNotEmpty } from '@rudderstack/integrations-lib';
 import { Context } from 'koa';
 import { ServiceSelector } from '../helpers/serviceSelector';
 import { DeliveryTestService } from '../services/delivertTest/deliveryTest';
@@ -19,12 +16,13 @@ import {
 import { FixMe } from '../util/types';
 import tags from '../v0/util/tags';
 import { ControllerUtility } from './util';
+import logger from '../logger';
 
 const NON_DETERMINABLE = 'Non-determinable';
 
 export class DeliveryController {
   public static async deliverToDestination(ctx: Context) {
-    logger.debug('Native(Delivery):: Request to transformer::', ctx.request.body);
+    logger.debug('Native(Delivery):: Request to transformer for delivery::', ctx.request.body);
     let deliveryResponse: DeliveryV0Response;
     const requestMetadata = MiscService.getRequestMetadata(ctx);
     const deliveryRequest = ctx.request.body as ProxyV0Request;
@@ -54,12 +52,12 @@ export class DeliveryController {
     ctx.body = { output: deliveryResponse };
     ControllerUtility.deliveryPostProcess(ctx, deliveryResponse.status);
 
-    logger.debug('Native(Delivery):: Response from transformer::', ctx.body);
+    logger.debug('Native(Delivery):: Response from transformer after delivery::', ctx.body);
     return ctx;
   }
 
   public static async deliverToDestinationV1(ctx: Context) {
-    logger.debug('Native(Delivery):: Request to transformer::', ctx.request.body);
+    logger.debug('Native(Delivery):: Request to transformer for delivery::', ctx.request.body);
     let deliveryResponse: DeliveryV1Response;
     const requestMetadata = MiscService.getRequestMetadata(ctx);
     const deliveryRequest = ctx.request.body as ProxyV1Request;
@@ -116,7 +114,7 @@ export class DeliveryController {
     );
     ctx.body = { output: response };
     ControllerUtility.postProcess(ctx);
-    logger.debug('Native(Delivery-Test):: Response from transformer::', ctx.body);
+    logger.debug('Native(Delivery-Test):: Response from transformer after delivery::', ctx.body);
     return ctx;
   }
 }