Skip to content

Commit

Permalink
chore: filtered logging and log for all destinations
Browse files Browse the repository at this point in the history 
Signed-off-by: Sai Sankeerth <[email protected]>
  • Loading branch information
Sai Sankeerth committed Jun 19, 2024
1 parent dc8eae2 commit 2ead5a8
Show file tree
Hide file tree
Showing 4 changed files with 156 additions and 108 deletions.
128 changes: 113 additions & 15 deletions src/adapters/network.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ const lodash = require('lodash');
const http = require('http');
const https = require('https');
const axios = require('axios');
const log = require('../logger');
const logger = require('../logger');
const stats = require('../util/stats');
const { removeUndefinedValues } = require('../v0/util');
const { removeUndefinedValues, isDefinedAndNotNullAndNotEmpty } = require('../v0/util');
const { processAxiosResponse } = require('./utils/networkUtils');
// Only for tests
const { setResponsesForMockAxiosAdapter } = require('../../test/testHelper');
Expand Down Expand Up @@ -56,7 +56,7 @@ const fireOutgoingReqStats = ({
statusCode,
clientResponse,
}) => {
const logMetaInfo = log.getLogMetadata(metadata);
const logMetaInfo = logger.getLogMetadata(metadata);
stats.timing('outgoing_request_latency', startTime, {
...logMetaInfo,
feature,
Expand Down Expand Up @@ -118,6 +118,28 @@ const enhanceRequestOptions = (options) => {
return requestOptions;
};

const getResponseDetails = (clientResponse) => {
if (!clientResponse.success) {
const { response } = clientResponse;
// non 2xx status handling for axios response
if (response) {
const { data, status, headers } = response;
return {
body: data || '',
status: status || 500,
...(isDefinedAndNotNullAndNotEmpty(headers) ? { headers } : {}),
};
}
return {};

Check warning on line 133 in src/adapters/network.js

View check run for this annotation

Codecov / codecov/patch

src/adapters/network.js#L133 

Added line #L133 was not covered by tests
}
const { data, status, headers } = clientResponse.response;
return {
body: data || '',
status: status || 500,
...(isDefinedAndNotNullAndNotEmpty(headers) ? { headers } : {}),
};
};

/**
* sends an http request with underlying client, expects request options
* @param {*} options
Expand All @@ -128,14 +150,27 @@ const httpSend = async (options, statTags = {}) => {
// here the options argument K-Vs will take priority over the default options
const requestOptions = enhanceRequestOptions(options);

const startTime = new Date();
const { url, data, method } = requestOptions;
const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
logger.requestLog(`${commonMsg} request`, {
metadata: statTags?.metadata,
requestDetails: {
url,
body: options?.data,
method: options?.method || statTags?.requestMethod,
},
});
const startTime = new Date();
try {
const response = await axios(requestOptions);
clientResponse = { success: true, response };
} catch (err) {
clientResponse = { success: false, response: err };
} finally {
logger.responseLog(`${commonMsg} response`, {
metadata: statTags?.metadata,
responseDetails: getResponseDetails(clientResponse),
});
fireHTTPStats(clientResponse, startTime, statTags);
}

Expand All @@ -156,13 +191,26 @@ const httpGET = async (url, options, statTags = {}) => {
// here the options argument K-Vs will take priority over the default options
const requestOptions = enhanceRequestOptions(options);

const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
logger.requestLog(`${commonMsg} request`, {
metadata: statTags?.metadata,
requestDetails: {
url,
body: requestOptions?.data,
method: 'get',
},
});
const startTime = new Date();
try {
const response = await axios.get(url, requestOptions);
clientResponse = { success: true, response };
} catch (err) {
clientResponse = { success: false, response: err };
} finally {
logger.responseLog(`${commonMsg} response`, {
metadata: statTags?.metadata,
responseDetails: getResponseDetails(clientResponse),
});
fireHTTPStats(clientResponse, startTime, statTags);
}
setResponsesForMockAxiosAdapter({ url, options, method: 'GET' }, clientResponse);
Expand All @@ -182,13 +230,26 @@ const httpDELETE = async (url, options, statTags = {}) => {
// here the options argument K-Vs will take priority over the default options
const requestOptions = enhanceRequestOptions(options);

const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
logger.requestLog(`${commonMsg} request`, {
metadata: statTags?.metadata,
requestDetails: {
url,
body: requestOptions?.data,
method: 'delete',
},
});
const startTime = new Date();
try {
const response = await axios.delete(url, requestOptions);
clientResponse = { success: true, response };
} catch (err) {
clientResponse = { success: false, response: err };
} finally {
logger.responseLog(`${commonMsg} response`, {
metadata: statTags?.metadata,
responseDetails: getResponseDetails(clientResponse),
});
fireHTTPStats(clientResponse, startTime, statTags);
}
setResponsesForMockAxiosAdapter({ url, options, method: 'DELETE' }, clientResponse);
Expand All @@ -209,13 +270,26 @@ const httpPOST = async (url, data, options, statTags = {}) => {
// here the options argument K-Vs will take priority over the default options
const requestOptions = enhanceRequestOptions(options);

const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
logger.requestLog(`${commonMsg} request`, {
metadata: statTags?.metadata,
requestDetails: {
url,
body: data,
method: 'post',
},
});
const startTime = new Date();
try {
const response = await axios.post(url, data, requestOptions);
clientResponse = { success: true, response };
} catch (err) {
clientResponse = { success: false, response: err };
} finally {
logger.responseLog(`${commonMsg} response`, {
metadata: statTags?.metadata,
responseDetails: getResponseDetails(clientResponse),
});
fireHTTPStats(clientResponse, startTime, statTags);
}
setResponsesForMockAxiosAdapter({ url, data, options, method: 'POST' }, clientResponse);
Expand All @@ -235,14 +309,26 @@ const httpPUT = async (url, data, options, statTags = {}) => {
let clientResponse;
// here the options argument K-Vs will take priority over the default options
const requestOptions = enhanceRequestOptions(options);

const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
logger.requestLog(`${commonMsg} request`, {
metadata: statTags?.metadata,
requestDetails: {
url,
body: data,
method: 'put',
},
});
const startTime = new Date();
try {
const response = await axios.put(url, data, requestOptions);
clientResponse = { success: true, response };
} catch (err) {
clientResponse = { success: false, response: err };
} finally {
logger.responseLog(`${commonMsg} response`, {
metadata: statTags?.metadata,
responseDetails: getResponseDetails(clientResponse),
});
fireHTTPStats(clientResponse, startTime, statTags);
}
setResponsesForMockAxiosAdapter({ url, data, options, method: 'PUT' }, clientResponse);
Expand All @@ -262,14 +348,26 @@ const httpPATCH = async (url, data, options, statTags = {}) => {
let clientResponse;
// here the options argument K-Vs will take priority over the default options
const requestOptions = enhanceRequestOptions(options);

const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`;
logger.requestLog(`${commonMsg} request`, {

Check warning on line 352 in src/adapters/network.js

View check run for this annotation

Codecov / codecov/patch

src/adapters/network.js#L352 

Added line #L352 was not covered by tests
metadata: statTags?.metadata,
requestDetails: {
url,
body: data,
method: 'patch',
},
});
const startTime = new Date();
try {
const response = await axios.patch(url, data, requestOptions);
clientResponse = { success: true, response };
} catch (err) {
clientResponse = { success: false, response: err };
} finally {
logger.responseLog(`${commonMsg} response`, {

Check warning on line 367 in src/adapters/network.js

View check run for this annotation

Codecov / codecov/patch

src/adapters/network.js#L367 

Added line #L367 was not covered by tests
metadata: statTags?.metadata,
responseDetails: getResponseDetails(clientResponse),
});
fireHTTPStats(clientResponse, startTime, statTags);
}
setResponsesForMockAxiosAdapter({ url, data, options, method: 'PATCH' }, clientResponse);
Expand Down Expand Up @@ -352,7 +450,7 @@ const prepareProxyRequest = (request) => {
// TODO:
break;
default:
log.debug(`body format ${payloadFormat} not supported`);
logger.debug(`body format ${payloadFormat} not supported`);
}
// Ref: https://github.com/rudderlabs/rudder-server/blob/master/router/network.go#L164
headers['User-Agent'] = 'RudderLabs';
Expand Down Expand Up @@ -426,14 +524,14 @@ const proxyRequest = async (request, destType) => {
headers,
method,
};
log.requestLog(`[${destType.toUpperCase()}] delivering data`, {
metadata,
requestDetails: {
body: data,
url: endpoint,
method,
},
});
// log.requestLog(`[${destType.toUpperCase()}] delivering data`, {
// metadata,
// requestDetails: {
// body: data,
// url: endpoint,
// method,
// },
// });
const response = await httpSend(requestOptions, {
feature: 'proxy',
destType,
Expand Down
34 changes: 26 additions & 8 deletions src/logger.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ const { LOGLEVELS, structuredLogger } = require('@rudderstack/integrations-lib')

// LOGGER_IMPL can be `console` or `winston`
const loggerImpl = process.env.LOGGER_IMPL ?? 'winston';
const logDestIds = (process.env.LOG_DEST_IDS ?? '').split(',')?.map?.((s) => s?.trim?.()); // should be comma separated
const logWspIds = (process.env.LOG_WSP_IDS ?? '').split(',')?.map?.((s) => s?.trim?.()); // should be comma separated

let logLevel = process.env.LOG_LEVEL ?? 'error';

Expand Down Expand Up @@ -71,6 +73,24 @@ const formLogArgs = (args) => {
return [msg, ...otherArgs];
};

const isMetadataMatching = (m) => {
const isDestIdConfigured = logDestIds?.find?.((envDId) => envDId && envDId === m?.destinationId);
const isWspIdConfigured = logWspIds?.find?.(
(envWspId) => envWspId && envWspId === m?.workspaceId,
);
return Boolean(isDestIdConfigured || isWspIdConfigured);
};

const getMatchedMetadata = (metadata) => {
if (!Array.isArray(metadata)) {
if (isMetadataMatching(metadata)) {
return [metadata];
}
return [];
}
return metadata.filter((m) => isMetadataMatching(m));
};

/**
* Perform logging operation on logMethod passed
*
Expand Down Expand Up @@ -143,8 +163,9 @@ const error = (...args) => {

const requestLog = (identifierMsg, { metadata, requestDetails: { url, body, method } }) => {
const logger = getLogger();
if (LOGLEVELS[logLevel] === LOGLEVELS.warn) {
const reqLogArgs = [identifierMsg, { metadata, url, body, method }];
const filteredMetadata = getMatchedMetadata(metadata);
if (LOGLEVELS[logLevel] === LOGLEVELS.warn && filteredMetadata.length > 0) {
const reqLogArgs = [identifierMsg, { metadata: filteredMetadata, url, body, method }];
log(logger.warn, reqLogArgs);
}
};
Expand All @@ -154,8 +175,9 @@ const responseLog = (
{ metadata, responseDetails: { response: body, status, headers } },
) => {
const logger = getLogger();
if (LOGLEVELS[logLevel] === LOGLEVELS.warn) {
const resLogArgs = [identifierMsg, { metadata, body, status, headers }];
const filteredMetadata = getMatchedMetadata(metadata);
if (LOGLEVELS[logLevel] === LOGLEVELS.warn && filteredMetadata.length > 0) {
const resLogArgs = [identifierMsg, { metadata: filteredMetadata, body, status, headers }];
log(logger.warn, resLogArgs);
}
};
Expand All @@ -166,10 +188,6 @@ module.exports = {
warn,
error,
setLogLevel,
// levelDebug,
// levelInfo,
// levelWarn,
// levelError,
responseLog,
getLogMetadata,
requestLog,
Expand Down
Loading

0 comments on commit 2ead5a8

Please sign in to comment.