chore: filtered logging and log for all destinations

src/adapters/network.js

@@ -5,9 +5,9 @@
 const http = require('http');
 const https = require('https');
 const axios = require('axios');
-const log = require('../logger');
+const logger = require('../logger');
 const stats = require('../util/stats');
-const { removeUndefinedValues } = require('../v0/util');
+const { removeUndefinedValues, isDefinedAndNotNullAndNotEmpty } = require('../v0/util');
 const { processAxiosResponse } = require('./utils/networkUtils');
 // Only for tests
 const { setResponsesForMockAxiosAdapter } = require('../../test/testHelper');
@@ -56,7 +56,7 @@
   statusCode,
   clientResponse,
 }) => {
-  const logMetaInfo = log.getLogMetadata(metadata);
+  const logMetaInfo = logger.getLogMetadata(metadata);
   stats.timing('outgoing_request_latency', startTime, {
     ...logMetaInfo,
     feature,
@@ -118,6 +118,28 @@
   return requestOptions;
 };
 
+const getResponseDetails = (clientResponse) => {
+  if (!clientResponse.success) {
+    const { response } = clientResponse;
+    // non 2xx status handling for axios response
+    if (response) {
+      const { data, status, headers } = response;
+      return {
+        body: data || '',
+        status: status || 500,
+        ...(isDefinedAndNotNullAndNotEmpty(headers) ? { headers } : {}),
+      };
+    }
+    return {};
+  }
+  const { data, status, headers } = clientResponse.response;
+  return {
+    body: data || '',
+    status: status || 500,
+    ...(isDefinedAndNotNullAndNotEmpty(headers) ? { headers } : {}),
+  };
+};
+
 /**
  * sends an http request with underlying client, expects request options
  * @param {*} options
@@ -128,14 +150,27 @@
   // here the options argument K-Vs will take priority over the default options
   const requestOptions = enhanceRequestOptions(options);
 
-  const startTime = new Date();
   const { url, data, method } = requestOptions;
+  const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
+  logger.requestLog(`${commonMsg} request`, {
+    metadata: statTags?.metadata,
+    requestDetails: {
+      url,
+      body: options?.data,
+      method: options?.method || statTags?.requestMethod,
+    },
+  });
+  const startTime = new Date();
   try {
     const response = await axios(requestOptions);
     clientResponse = { success: true, response };
   } catch (err) {
     clientResponse = { success: false, response: err };
   } finally {
+    logger.responseLog(`${commonMsg} response`, {
+      metadata: statTags?.metadata,
+      responseDetails: getResponseDetails(clientResponse),
+    });
     fireHTTPStats(clientResponse, startTime, statTags);
   }
 
@@ -156,13 +191,26 @@
   // here the options argument K-Vs will take priority over the default options
   const requestOptions = enhanceRequestOptions(options);
 
+  const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
+  logger.requestLog(`${commonMsg} request`, {
+    metadata: statTags?.metadata,
+    requestDetails: {
+      url,
+      body: requestOptions?.data,
+      method: 'get',
+    },
+  });
   const startTime = new Date();
   try {
     const response = await axios.get(url, requestOptions);
     clientResponse = { success: true, response };
   } catch (err) {
     clientResponse = { success: false, response: err };
   } finally {
+    logger.responseLog(`${commonMsg} response`, {
+      metadata: statTags?.metadata,
+      responseDetails: getResponseDetails(clientResponse),
+    });
     fireHTTPStats(clientResponse, startTime, statTags);
   }
   setResponsesForMockAxiosAdapter({ url, options, method: 'GET' }, clientResponse);
@@ -182,13 +230,26 @@
   // here the options argument K-Vs will take priority over the default options
   const requestOptions = enhanceRequestOptions(options);
 
+  const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
+  logger.requestLog(`${commonMsg} request`, {
+    metadata: statTags?.metadata,
+    requestDetails: {
+      url,
+      body: requestOptions?.data,
+      method: 'delete',
+    },
+  });
   const startTime = new Date();
   try {
     const response = await axios.delete(url, requestOptions);
     clientResponse = { success: true, response };
   } catch (err) {
     clientResponse = { success: false, response: err };
   } finally {
+    logger.responseLog(`${commonMsg} response`, {
+      metadata: statTags?.metadata,
+      responseDetails: getResponseDetails(clientResponse),
+    });
     fireHTTPStats(clientResponse, startTime, statTags);
   }
   setResponsesForMockAxiosAdapter({ url, options, method: 'DELETE' }, clientResponse);
@@ -209,13 +270,26 @@
   // here the options argument K-Vs will take priority over the default options
   const requestOptions = enhanceRequestOptions(options);
 
+  const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
+  logger.requestLog(`${commonMsg} request`, {
+    metadata: statTags?.metadata,
+    requestDetails: {
+      url,
+      body: data,
+      method: 'post',
+    },
+  });
   const startTime = new Date();
   try {
     const response = await axios.post(url, data, requestOptions);
     clientResponse = { success: true, response };
   } catch (err) {
     clientResponse = { success: false, response: err };
   } finally {
+    logger.responseLog(`${commonMsg} response`, {
+      metadata: statTags?.metadata,
+      responseDetails: getResponseDetails(clientResponse),
+    });
     fireHTTPStats(clientResponse, startTime, statTags);
   }
   setResponsesForMockAxiosAdapter({ url, data, options, method: 'POST' }, clientResponse);
@@ -235,14 +309,26 @@
   let clientResponse;
   // here the options argument K-Vs will take priority over the default options
   const requestOptions = enhanceRequestOptions(options);
-
+  const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
+  logger.requestLog(`${commonMsg} request`, {
+    metadata: statTags?.metadata,
+    requestDetails: {
+      url,
+      body: data,
+      method: 'put',
+    },
+  });
   const startTime = new Date();
   try {
     const response = await axios.put(url, data, requestOptions);
     clientResponse = { success: true, response };
   } catch (err) {
     clientResponse = { success: false, response: err };
   } finally {
+    logger.responseLog(`${commonMsg} response`, {
+      metadata: statTags?.metadata,
+      responseDetails: getResponseDetails(clientResponse),
+    });
     fireHTTPStats(clientResponse, startTime, statTags);
   }
   setResponsesForMockAxiosAdapter({ url, data, options, method: 'PUT' }, clientResponse);
@@ -262,14 +348,26 @@
   let clientResponse;
   // here the options argument K-Vs will take priority over the default options
   const requestOptions = enhanceRequestOptions(options);
-
+  const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
+  logger.requestLog(`${commonMsg} request`, {
+    metadata: statTags?.metadata,
+    requestDetails: {
+      url,
+      body: data,
+      method: 'patch',
+    },
+  });
   const startTime = new Date();
   try {
     const response = await axios.patch(url, data, requestOptions);
     clientResponse = { success: true, response };
   } catch (err) {
     clientResponse = { success: false, response: err };
   } finally {
+    logger.responseLog(`${commonMsg} response`, {
+      metadata: statTags?.metadata,
+      responseDetails: getResponseDetails(clientResponse),
+    });
     fireHTTPStats(clientResponse, startTime, statTags);
   }
   setResponsesForMockAxiosAdapter({ url, data, options, method: 'PATCH' }, clientResponse);
@@ -352,7 +450,7 @@
       // TODO:
       break;
     default:
-      log.debug(`body format ${payloadFormat} not supported`);
+      logger.debug(`body format ${payloadFormat} not supported`);
   }
   // Ref: https://github.com/rudderlabs/rudder-server/blob/master/router/network.go#L164
   headers['User-Agent'] = 'RudderLabs';
@@ -426,14 +524,14 @@
     headers,
     method,
   };
-  log.requestLog(`[${destType.toUpperCase()}] delivering data`, {
-    metadata,
-    requestDetails: {
-      body: data,
-      url: endpoint,
-      method,
-    },
-  });
+  // log.requestLog(`[${destType.toUpperCase()}] delivering data`, {
+  //   metadata,
+  //   requestDetails: {
+  //     body: data,
+  //     url: endpoint,
+  //     method,
+  //   },
+  // });
   const response = await httpSend(requestOptions, {
     feature: 'proxy',
     destType,

src/logger.js

@@ -3,6 +3,8 @@ const { LOGLEVELS, structuredLogger } = require('@rudderstack/integrations-lib')
 
 // LOGGER_IMPL can be `console` or `winston`
 const loggerImpl = process.env.LOGGER_IMPL ?? 'winston';
+const logDestIds = (process.env.LOG_DEST_IDS ?? '').split(',')?.map?.((s) => s?.trim?.()); // should be comma separated
+const logWspIds = (process.env.LOG_WSP_IDS ?? '').split(',')?.map?.((s) => s?.trim?.()); // should be comma separated
 
 let logLevel = process.env.LOG_LEVEL ?? 'error';
 
@@ -71,6 +73,24 @@ const formLogArgs = (args) => {
   return [msg, ...otherArgs];
 };
 
+const isMetadataMatching = (m) => {
+  const isDestIdConfigured = logDestIds?.find?.((envDId) => envDId && envDId === m?.destinationId);
+  const isWspIdConfigured = logWspIds?.find?.(
+    (envWspId) => envWspId && envWspId === m?.workspaceId,
+  );
+  return Boolean(isDestIdConfigured || isWspIdConfigured);
+};
+
+const getMatchedMetadata = (metadata) => {
+  if (!Array.isArray(metadata)) {
+    if (isMetadataMatching(metadata)) {
+      return [metadata];
+    }
+    return [];
+  }
+  return metadata.filter((m) => isMetadataMatching(m));
+};
+
 /**
  * Perform logging operation on logMethod passed
  *
@@ -143,8 +163,9 @@ const error = (...args) => {
 
 const requestLog = (identifierMsg, { metadata, requestDetails: { url, body, method } }) => {
   const logger = getLogger();
-  if (LOGLEVELS[logLevel] === LOGLEVELS.warn) {
-    const reqLogArgs = [identifierMsg, { metadata, url, body, method }];
+  const filteredMetadata = getMatchedMetadata(metadata);
+  if (LOGLEVELS[logLevel] === LOGLEVELS.warn && filteredMetadata.length > 0) {
+    const reqLogArgs = [identifierMsg, { metadata: filteredMetadata, url, body, method }];
     log(logger.warn, reqLogArgs);
   }
 };
@@ -154,8 +175,9 @@ const responseLog = (
   { metadata, responseDetails: { response: body, status, headers } },
 ) => {
   const logger = getLogger();
-  if (LOGLEVELS[logLevel] === LOGLEVELS.warn) {
-    const resLogArgs = [identifierMsg, { metadata, body, status, headers }];
+  const filteredMetadata = getMatchedMetadata(metadata);
+  if (LOGLEVELS[logLevel] === LOGLEVELS.warn && filteredMetadata.length > 0) {
+    const resLogArgs = [identifierMsg, { metadata: filteredMetadata, body, status, headers }];
     log(logger.warn, resLogArgs);
   }
 };
@@ -166,10 +188,6 @@ module.exports = {
   warn,
   error,
   setLogLevel,
-  // levelDebug,
-  // levelInfo,
-  // levelWarn,
-  // levelError,
   responseLog,
   getLogMetadata,
   requestLog,