diff --git a/src/adapters/network.js b/src/adapters/network.js index 850ba649c8..92411deab1 100644 --- a/src/adapters/network.js +++ b/src/adapters/network.js @@ -5,9 +5,9 @@ const lodash = require('lodash'); const http = require('http'); const https = require('https'); const axios = require('axios'); -const log = require('../logger'); +const logger = require('../logger'); const stats = require('../util/stats'); -const { removeUndefinedValues } = require('../v0/util'); +const { removeUndefinedValues, isDefinedAndNotNullAndNotEmpty } = require('../v0/util'); const { processAxiosResponse } = require('./utils/networkUtils'); // Only for tests const { setResponsesForMockAxiosAdapter } = require('../../test/testHelper'); @@ -56,7 +56,7 @@ const fireOutgoingReqStats = ({ statusCode, clientResponse, }) => { - const logMetaInfo = log.getLogMetadata(metadata); + const logMetaInfo = logger.getLogMetadata(metadata); stats.timing('outgoing_request_latency', startTime, { ...logMetaInfo, feature, @@ -118,6 +118,28 @@ const enhanceRequestOptions = (options) => { return requestOptions; }; +const getResponseDetails = (clientResponse) => { + if (!clientResponse.success) { + const { response } = clientResponse; + // non 2xx status handling for axios response + if (response) { + const { data, status, headers } = response; + return { + body: data || '', + status: status || 500, + ...(isDefinedAndNotNullAndNotEmpty(headers) ? { headers } : {}), + }; + } + return {}; + } + const { data, status, headers } = clientResponse.response; + return { + body: data || '', + status: status || 500, + ...(isDefinedAndNotNullAndNotEmpty(headers) ? { headers } : {}), + }; +}; + /** * sends an http request with underlying client, expects request options * @param {*} options @@ -128,14 +150,27 @@ const httpSend = async (options, statTags = {}) => { // here the options argument K-Vs will take priority over the default options const requestOptions = enhanceRequestOptions(options); - const startTime = new Date(); const { url, data, method } = requestOptions; + const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`; + logger.requestLog(`${commonMsg} request`, { + metadata: statTags?.metadata, + requestDetails: { + url, + body: options?.data, + method: options?.method || statTags?.requestMethod, + }, + }); + const startTime = new Date(); try { const response = await axios(requestOptions); clientResponse = { success: true, response }; } catch (err) { clientResponse = { success: false, response: err }; } finally { + logger.responseLog(`${commonMsg} response`, { + metadata: statTags?.metadata, + responseDetails: getResponseDetails(clientResponse), + }); fireHTTPStats(clientResponse, startTime, statTags); } @@ -156,6 +191,15 @@ const httpGET = async (url, options, statTags = {}) => { // here the options argument K-Vs will take priority over the default options const requestOptions = enhanceRequestOptions(options); + const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`; + logger.requestLog(`${commonMsg} request`, { + metadata: statTags?.metadata, + requestDetails: { + url, + body: requestOptions?.data, + method: 'get', + }, + }); const startTime = new Date(); try { const response = await axios.get(url, requestOptions); @@ -163,6 +207,10 @@ const httpGET = async (url, options, statTags = {}) => { } catch (err) { clientResponse = { success: false, response: err }; } finally { + logger.responseLog(`${commonMsg} response`, { + metadata: statTags?.metadata, + responseDetails: getResponseDetails(clientResponse), + }); fireHTTPStats(clientResponse, startTime, statTags); } setResponsesForMockAxiosAdapter({ url, options, method: 'GET' }, clientResponse); @@ -182,6 +230,15 @@ const httpDELETE = async (url, options, statTags = {}) => { // here the options argument K-Vs will take priority over the default options const requestOptions = enhanceRequestOptions(options); + const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`; + logger.requestLog(`${commonMsg} request`, { + metadata: statTags?.metadata, + requestDetails: { + url, + body: requestOptions?.data, + method: 'delete', + }, + }); const startTime = new Date(); try { const response = await axios.delete(url, requestOptions); @@ -189,6 +246,10 @@ const httpDELETE = async (url, options, statTags = {}) => { } catch (err) { clientResponse = { success: false, response: err }; } finally { + logger.responseLog(`${commonMsg} response`, { + metadata: statTags?.metadata, + responseDetails: getResponseDetails(clientResponse), + }); fireHTTPStats(clientResponse, startTime, statTags); } setResponsesForMockAxiosAdapter({ url, options, method: 'DELETE' }, clientResponse); @@ -209,6 +270,15 @@ const httpPOST = async (url, data, options, statTags = {}) => { // here the options argument K-Vs will take priority over the default options const requestOptions = enhanceRequestOptions(options); + const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`; + logger.requestLog(`${commonMsg} request`, { + metadata: statTags?.metadata, + requestDetails: { + url, + body: data, + method: 'post', + }, + }); const startTime = new Date(); try { const response = await axios.post(url, data, requestOptions); @@ -216,6 +286,10 @@ const httpPOST = async (url, data, options, statTags = {}) => { } catch (err) { clientResponse = { success: false, response: err }; } finally { + logger.responseLog(`${commonMsg} response`, { + metadata: statTags?.metadata, + responseDetails: getResponseDetails(clientResponse), + }); fireHTTPStats(clientResponse, startTime, statTags); } setResponsesForMockAxiosAdapter({ url, data, options, method: 'POST' }, clientResponse); @@ -235,7 +309,15 @@ const httpPUT = async (url, data, options, statTags = {}) => { let clientResponse; // here the options argument K-Vs will take priority over the default options const requestOptions = enhanceRequestOptions(options); - + const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`; + logger.requestLog(`${commonMsg} request`, { + metadata: statTags?.metadata, + requestDetails: { + url, + body: data, + method: 'put', + }, + }); const startTime = new Date(); try { const response = await axios.put(url, data, requestOptions); @@ -243,6 +325,10 @@ const httpPUT = async (url, data, options, statTags = {}) => { } catch (err) { clientResponse = { success: false, response: err }; } finally { + logger.responseLog(`${commonMsg} response`, { + metadata: statTags?.metadata, + responseDetails: getResponseDetails(clientResponse), + }); fireHTTPStats(clientResponse, startTime, statTags); } setResponsesForMockAxiosAdapter({ url, data, options, method: 'PUT' }, clientResponse); @@ -262,7 +348,15 @@ const httpPATCH = async (url, data, options, statTags = {}) => { let clientResponse; // here the options argument K-Vs will take priority over the default options const requestOptions = enhanceRequestOptions(options); - + const commonMsg = `[${statTags?.destType?.toUpperCase?.() || ''}] ${statTags?.endpointPath}`; + logger.requestLog(`${commonMsg} request`, { + metadata: statTags?.metadata, + requestDetails: { + url, + body: data, + method: 'patch', + }, + }); const startTime = new Date(); try { const response = await axios.patch(url, data, requestOptions); @@ -270,6 +364,10 @@ const httpPATCH = async (url, data, options, statTags = {}) => { } catch (err) { clientResponse = { success: false, response: err }; } finally { + logger.responseLog(`${commonMsg} response`, { + metadata: statTags?.metadata, + responseDetails: getResponseDetails(clientResponse), + }); fireHTTPStats(clientResponse, startTime, statTags); } setResponsesForMockAxiosAdapter({ url, data, options, method: 'PATCH' }, clientResponse); @@ -352,7 +450,7 @@ const prepareProxyRequest = (request) => { // TODO: break; default: - log.debug(`body format ${payloadFormat} not supported`); + logger.debug(`body format ${payloadFormat} not supported`); } // Ref: https://github.com/rudderlabs/rudder-server/blob/master/router/network.go#L164 headers['User-Agent'] = 'RudderLabs'; @@ -426,14 +524,14 @@ const proxyRequest = async (request, destType) => { headers, method, }; - log.requestLog(`[${destType.toUpperCase()}] delivering data`, { - metadata, - requestDetails: { - body: data, - url: endpoint, - method, - }, - }); + // log.requestLog(`[${destType.toUpperCase()}] delivering data`, { + // metadata, + // requestDetails: { + // body: data, + // url: endpoint, + // method, + // }, + // }); const response = await httpSend(requestOptions, { feature: 'proxy', destType, diff --git a/src/logger.js b/src/logger.js index 6daff56c67..edea85284e 100644 --- a/src/logger.js +++ b/src/logger.js @@ -3,6 +3,8 @@ const { LOGLEVELS, structuredLogger } = require('@rudderstack/integrations-lib') // LOGGER_IMPL can be `console` or `winston` const loggerImpl = process.env.LOGGER_IMPL ?? 'winston'; +const logDestIds = (process.env.LOG_DEST_IDS ?? '').split(',')?.map?.((s) => s?.trim?.()); // should be comma separated +const logWspIds = (process.env.LOG_WSP_IDS ?? '').split(',')?.map?.((s) => s?.trim?.()); // should be comma separated let logLevel = process.env.LOG_LEVEL ?? 'error'; @@ -71,6 +73,24 @@ const formLogArgs = (args) => { return [msg, ...otherArgs]; }; +const isMetadataMatching = (m) => { + const isDestIdConfigured = logDestIds?.find?.((envDId) => envDId && envDId === m?.destinationId); + const isWspIdConfigured = logWspIds?.find?.( + (envWspId) => envWspId && envWspId === m?.workspaceId, + ); + return Boolean(isDestIdConfigured || isWspIdConfigured); +}; + +const getMatchedMetadata = (metadata) => { + if (!Array.isArray(metadata)) { + if (isMetadataMatching(metadata)) { + return [metadata]; + } + return []; + } + return metadata.filter((m) => isMetadataMatching(m)); +}; + /** * Perform logging operation on logMethod passed * @@ -143,8 +163,9 @@ const error = (...args) => { const requestLog = (identifierMsg, { metadata, requestDetails: { url, body, method } }) => { const logger = getLogger(); - if (LOGLEVELS[logLevel] === LOGLEVELS.warn) { - const reqLogArgs = [identifierMsg, { metadata, url, body, method }]; + const filteredMetadata = getMatchedMetadata(metadata); + if (LOGLEVELS[logLevel] === LOGLEVELS.warn && filteredMetadata.length > 0) { + const reqLogArgs = [identifierMsg, { metadata: filteredMetadata, url, body, method }]; log(logger.warn, reqLogArgs); } }; @@ -154,8 +175,9 @@ const responseLog = ( { metadata, responseDetails: { response: body, status, headers } }, ) => { const logger = getLogger(); - if (LOGLEVELS[logLevel] === LOGLEVELS.warn) { - const resLogArgs = [identifierMsg, { metadata, body, status, headers }]; + const filteredMetadata = getMatchedMetadata(metadata); + if (LOGLEVELS[logLevel] === LOGLEVELS.warn && filteredMetadata.length > 0) { + const resLogArgs = [identifierMsg, { metadata: filteredMetadata, body, status, headers }]; log(logger.warn, resLogArgs); } }; @@ -166,10 +188,6 @@ module.exports = { warn, error, setLogLevel, - // levelDebug, - // levelInfo, - // levelWarn, - // levelError, responseLog, getLogMetadata, requestLog, diff --git a/src/v0/destinations/google_adwords_offline_conversions/networkHandler.js b/src/v0/destinations/google_adwords_offline_conversions/networkHandler.js index 51bc57d176..f0486c3b93 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/networkHandler.js +++ b/src/v0/destinations/google_adwords_offline_conversions/networkHandler.js @@ -27,14 +27,6 @@ const conversionCustomVariableCache = new Cache(CONVERSION_CUSTOM_VARIABLE_CACHE const createJob = async ({ endpoint, headers, payload, metadata }) => { const endPoint = `${endpoint}:create`; - logger.requestLog(`[${destType.toUpperCase()}] job creation request`, { - metadata, - requestDetails: { - url: endpoint, - body: payload, - method: 'post', - }, - }); let createJobResponse = await httpPOST( endPoint, payload, @@ -49,15 +41,7 @@ const createJob = async ({ endpoint, headers, payload, metadata }) => { }, ); createJobResponse = processAxiosResponse(createJobResponse); - const { response, status, headers: responseHeaders } = createJobResponse; - logger.responseLog(`[${destType.toUpperCase()}] create job`, { - metadata, - responseDetails: { - headers: responseHeaders, - status, - response, - }, - }); + const { response, status } = createJobResponse; if (!isHttpStatusSuccess(status)) { throw new AbortedError( `[Google Ads Offline Conversions]:: ${response?.error?.message} during google_ads_offline_store_conversions Job Creation`, @@ -71,14 +55,6 @@ const createJob = async ({ endpoint, headers, payload, metadata }) => { const addConversionToJob = async ({ endpoint, headers, jobId, payload, metadata }) => { const endPoint = `${endpoint}/${jobId}:addOperations`; - logger.requestLog(`[${destType.toUpperCase()}] add conversion to job request`, { - metadata, - requestDetails: { - url: endpoint, - body: payload, - method: 'post', - }, - }); let addConversionToJobResponse = await httpPOST( endPoint, payload, @@ -93,15 +69,7 @@ const addConversionToJob = async ({ endpoint, headers, jobId, payload, metadata }, ); addConversionToJobResponse = processAxiosResponse(addConversionToJobResponse); - const { response, status, headers: responseHeaders } = addConversionToJobResponse; - logger.responseLog(`[${destType.toUpperCase()}] add conversion to job`, { - metadata, - responseDetails: { - response, - status, - headers: responseHeaders, - }, - }); + const { response, status } = addConversionToJobResponse; if (!isHttpStatusSuccess(status)) { throw new AbortedError( `[Google Ads Offline Conversions]:: ${response?.error?.message} during google_ads_offline_store_conversions Add Conversion`, @@ -115,15 +83,7 @@ const addConversionToJob = async ({ endpoint, headers, jobId, payload, metadata const runTheJob = async ({ endpoint, headers, payload, jobId, metadata }) => { const endPoint = `${endpoint}/${jobId}:run`; - logger.requestLog(`[${destType.toUpperCase()}] run job request`, { - metadata, - requestDetails: { - body: payload, - method: 'POST', - url: endPoint, - }, - }); - const { httpResponse: executeJobResponse, processedResponse } = await handleHttpRequest( + const { httpResponse: executeJobResponse } = await handleHttpRequest( 'post', endPoint, payload, @@ -137,15 +97,6 @@ const runTheJob = async ({ endpoint, headers, payload, jobId, metadata }) => { metadata, }, ); - const { headers: responseHeaders, response, status } = processedResponse; - logger.responseLog(`[${destType.toUpperCase()}] run job`, { - metadata, - responseDetails: { - response, - status, - responseHeaders, - }, - }); return executeJobResponse; }; @@ -339,11 +290,11 @@ const ProxyRequest = async (request) => { } } const requestBody = { url: endpoint, data: body.JSON, headers, method }; - logger.requestLog(`[${destType.toUpperCase()}] offline conversion creation request`, { - metadata, - requestDetails: { url: requestBody.url, body: requestBody.data, method }, - }); - const { httpResponse, processedResponse } = await handleHttpRequest('constructor', requestBody, { + // logger.requestLog(`[${destType.toUpperCase()}] offline conversion creation request`, { + // metadata, + // requestDetails: { url: requestBody.url, body: requestBody.data, method }, + // }); + const { httpResponse } = await handleHttpRequest('constructor', requestBody, { feature: 'proxy', destType: 'gogole_adwords_offline_conversions', endpointPath: `/proxy`, @@ -351,15 +302,14 @@ const ProxyRequest = async (request) => { module: 'dataDelivery', metadata, }); - const { headers: responseHeaders, status, response } = processedResponse; - logger.responseLog(`[${destType.toUpperCase()}] deliver event to destination`, { - metadata, - responseDetails: { - response, - headers: responseHeaders, - status, - }, - }); + // logger.responseLog(`[${destType.toUpperCase()}] deliver event to destination`, { + // metadata, + // responseDetails: { + // response, + // headers: responseHeaders, + // status, + // }, + // }); return httpResponse; }; diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js index bf1773d450..7c21371ccf 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.js @@ -30,13 +30,11 @@ const { CLICK_CONVERSION, trackCallConversionsMapping, consentConfigMap, - destType, } = require('./config'); const { processAxiosResponse } = require('../../../adapters/utils/networkUtils'); const Cache = require('../../util/cache'); const helper = require('./helper'); const { finaliseConsent } = require('../../util/googleUtils'); -const logger = require('../../../logger'); const conversionActionIdCache = new Cache(CONVERSION_ACTION_ID_CACHE_TTL); @@ -71,14 +69,6 @@ const getConversionActionId = async ({ headers, params, metadata }) => { const requestOptions = { headers, }; - logger.requestLog(`[${destType.toUpperCase()}] get conversion action id request`, { - metadata, - requestDetails: { - url: endpoint, - body: data, - method: 'post', - }, - }); let searchStreamResponse = await httpPOST(endpoint, data, requestOptions, { destType: 'google_adwords_offline_conversions', feature: 'transformation', @@ -88,15 +78,7 @@ const getConversionActionId = async ({ headers, params, metadata }) => { metadata, }); searchStreamResponse = processAxiosResponse(searchStreamResponse); - const { response, status, headers: responseHeaders } = searchStreamResponse; - logger.responseLog(`[${destType.toUpperCase()}] get conversion action id response`, { - metadata, - responseDetails: { - response, - status, - headers: responseHeaders, - }, - }); + const { response, status } = searchStreamResponse; if (!isHttpStatusSuccess(status)) { throw new AbortedError( `[Google Ads Offline Conversions]:: ${JSON.stringify(