From 9b829315ae5138ed56328fbf740b0f5bcf4f4e21 Mon Sep 17 00:00:00 2001 From: Jayachand Date: Wed, 10 Jul 2024 12:59:44 +0530 Subject: [PATCH] chore: cache dns resolution in node (#3495) * chore: cache dns resolution in node --- src/util/prometheus.js | 2 +- src/util/utils.js | 48 ++++++++++++++----- .../user_transformation_fetch.test.js | 12 ++--- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/src/util/prometheus.js b/src/util/prometheus.js index 0854835c98..0b1f3b34bf 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -892,7 +892,7 @@ class Prometheus { name: 'fetch_dns_resolve_time', help: 'fetch_dns_resolve_time', type: 'histogram', - labelNames: ['identifier', 'transformationId', 'workspaceId', 'error'], + labelNames: ['identifier', 'transformationId', 'workspaceId', 'error', 'cacheHit'], }, { name: 'geo_call_duration', diff --git a/src/util/utils.js b/src/util/utils.js index eb5a011444..82c85b41a5 100644 --- a/src/util/utils.js +++ b/src/util/utils.js @@ -5,6 +5,7 @@ const { Resolver } = require('dns').promises; const fetch = require('node-fetch'); const util = require('util'); +const NodeCache = require('node-cache'); const logger = require('../logger'); const stats = require('./stats'); @@ -15,12 +16,41 @@ const BLOCK_HOST_NAMES_LIST = BLOCK_HOST_NAMES.split(','); const LOCAL_HOST_NAMES_LIST = ['localhost', '127.0.0.1', '[::]', '[::1]']; const LOCALHOST_OCTET = '127.'; const RECORD_TYPE_A = 4; // ipv4 +const DNS_CACHE_ENABLED = process.env.DNS_CACHE_ENABLED === 'true'; +const DNS_CACHE_TTL = process.env.DNS_CACHE_TTL ? parseInt(process.env.DNS_CACHE_TTL, 10) : 300; +const dnsCache = new NodeCache({ + useClones: false, + stdTTL: DNS_CACHE_TTL, + checkperiod: DNS_CACHE_TTL, +}); + +const resolveHostName = async (hostname) => { + // ex: [{ address: '108.157.0.0', ttl: 600 }] + const addresses = await resolver.resolve4(hostname, { ttl: true }); + return addresses.length > 0 ? addresses[0] : {}; +}; + +const fetchAddressFromHostName = async (hostname) => { + if (!DNS_CACHE_ENABLED) { + const { address } = await resolveHostName(hostname); + return { address, cacheHit: false }; + } + const cachedAddress = dnsCache.get(hostname); + if (cachedAddress !== undefined) { + return { address: cachedAddress, cacheHit: true }; + } + const { address, ttl } = await resolveHostName(hostname); + dnsCache.set(hostname, address, Math.min(ttl, DNS_CACHE_TTL)); + return { address, cacheHit: false }; +}; const staticLookup = (transformationTags) => async (hostname, _, cb) => { - let ips; + let ip; const resolveStartTime = new Date(); try { - ips = await resolver.resolve4(hostname); + const { address, cacheHit } = await fetchAddressFromHostName(hostname); + ip = address; + stats.timing('fetch_dns_resolve_time', resolveStartTime, { ...transformationTags, cacheHit }); } catch (error) { logger.error(`DNS Error Code: ${error.code} | Message : ${error.message}`); stats.timing('fetch_dns_resolve_time', resolveStartTime, { @@ -30,22 +60,18 @@ const staticLookup = (transformationTags) => async (hostname, _, cb) => { cb(null, `unable to resolve IP address for ${hostname}`, RECORD_TYPE_A); return; } - stats.timing('fetch_dns_resolve_time', resolveStartTime, transformationTags); - if (ips.length === 0) { + if (!ip) { cb(null, `resolved empty list of IP address for ${hostname}`, RECORD_TYPE_A); return; } - // eslint-disable-next-line no-restricted-syntax - for (const ip of ips) { - if (ip.startsWith(LOCALHOST_OCTET)) { - cb(null, `cannot use ${ip} as IP address`, RECORD_TYPE_A); - return; - } + if (ip.startsWith(LOCALHOST_OCTET)) { + cb(null, `cannot use ${ip} as IP address`, RECORD_TYPE_A); + return; } - cb(null, ips[0], RECORD_TYPE_A); + cb(null, ip, RECORD_TYPE_A); }; const httpAgentWithDnsLookup = (scheme, transformationTags) => { diff --git a/test/__tests__/user_transformation_fetch.test.js b/test/__tests__/user_transformation_fetch.test.js index e37b42ea1e..3f0b060689 100644 --- a/test/__tests__/user_transformation_fetch.test.js +++ b/test/__tests__/user_transformation_fetch.test.js @@ -91,11 +91,11 @@ describe("User transformation fetch tests", () => { }; const errMsg = "ERROR"; - mockResolver.mockResolvedValue([ '127.0.0.1' ]); + mockResolver.mockResolvedValue([{ address: '127.0.0.1', ttl: 300 } ]); const output = await userTransformHandler(inputData, versionId, [], trRevCode, true); expect(mockResolver).toHaveBeenCalledTimes(inputData.length); - expect(mockResolver).toHaveBeenCalledWith('abc.xyz.com'); + expect(mockResolver).toHaveBeenCalledWith('abc.xyz.com', { ttl: true }); output.transformedEvents.forEach(ev => { expect(ev.errMsg).toEqual(errMsg); }); @@ -155,7 +155,7 @@ describe("User transformation fetch tests", () => { const output = await userTransformHandler(inputData, versionId, [], trRevCode, true); expect(mockResolver).toHaveBeenCalledTimes(inputData.length); - expect(mockResolver).toHaveBeenCalledWith('abc.xyz.com'); + expect(mockResolver).toHaveBeenCalledWith('abc.xyz.com', { ttl: true }); output.transformedEvents.forEach(ev => { expect(ev.errMsg).toEqual(errMsg); }); @@ -253,7 +253,7 @@ describe("User transformation fetch tests", () => { const output = await userTransformHandler(inputData, versionId, [], trRevCode, true); expect(mockResolver).toHaveBeenCalledTimes(inputData.length); - expect(mockResolver).toHaveBeenCalledWith('abc.xyz.com'); + expect(mockResolver).toHaveBeenCalledWith('abc.xyz.com', { ttl: true }); output.transformedEvents.forEach(ev => { expect(ev.errMsg).toEqual(errMsg); }); @@ -307,11 +307,11 @@ describe("User transformation fetch tests", () => { }; const errMsg = "request to https://abc.xyz.com/dummyUrl failed, reason: Invalid IP address: cannot use 127.0.0.1 as IP address"; - mockResolver.mockResolvedValue(['3.122.122.122', '127.0.0.1']); + mockResolver.mockResolvedValue([{ address: '127.0.0.1', ttl: 100 }, { address: '3.122.122.122', ttl: 600 }]); const output = await userTransformHandler(inputData, versionId, [], trRevCode, true); expect(mockResolver).toHaveBeenCalledTimes(inputData.length); - expect(mockResolver).toHaveBeenCalledWith('abc.xyz.com'); + expect(mockResolver).toHaveBeenCalledWith('abc.xyz.com', { ttl: true }); output.transformedEvents.forEach(ev => { expect(ev.errMsg).toEqual(errMsg); });