From 0379c4d8d5cf779f84e7d44a6d061ce988290e2e Mon Sep 17 00:00:00 2001 From: Jayachand Date: Tue, 25 Jun 2024 17:41:12 +0530 Subject: [PATCH] chore: remove version id, clean unused stats (#3494) * chore: remove versionId, unused or unimportant stats --- src/legacy/router.js | 18 +- src/util/customTransformer-faas.js | 2 +- src/util/customTransformer-v1.js | 1 - src/util/customTransformer.js | 22 +- src/util/customTransforrmationsStore-v1.js | 37 +- src/util/customTransforrmationsStore.js | 8 +- src/util/ivmFactory.js | 18 +- src/util/prometheus.js | 558 ++++++++------------- src/util/utils.js | 14 +- 9 files changed, 254 insertions(+), 424 deletions(-) diff --git a/src/legacy/router.js b/src/legacy/router.js index afc8c1a797..043e37b66d 100644 --- a/src/legacy/router.js +++ b/src/legacy/router.js @@ -539,9 +539,7 @@ if (startDestTransformer) { (event) => `${event.metadata.destinationId}_${event.metadata.sourceId}`, ); } - stats.counter('user_transform_function_group_size', Object.entries(groupedEvents).length, { - processSessions, - }); + stats.counter('user_transform_function_group_size', Object.entries(groupedEvents).length, {}); let ctxStatusCode = 200; const transformedEvents = []; @@ -646,16 +644,10 @@ if (startDestTransformer) { ctx.status = ctxStatusCode; ctx.set('apiVersion', API_VERSION); - stats.timing('user_transform_request_latency', startTime, { - processSessions, - }); - stats.timingSummary('user_transform_request_latency_summary', startTime, { - processSessions, - }); - stats.increment('user_transform_requests', { processSessions }); - stats.histogram('user_transform_output_events', transformedEvents.length, { - processSessions, - }); + stats.timing('user_transform_request_latency', startTime, {}); + stats.timingSummary('user_transform_request_latency_summary', startTime, {}); + stats.increment('user_transform_requests', {}); + stats.histogram('user_transform_output_events', transformedEvents.length, {}); }); } } diff --git a/src/util/customTransformer-faas.js b/src/util/customTransformer-faas.js index 9ac9804097..07dc205582 100644 --- a/src/util/customTransformer-faas.js +++ b/src/util/customTransformer-faas.js @@ -91,7 +91,7 @@ async function setOpenFaasUserTransform( trMetadata = {}, ) { const tags = { - transformerVersionId: userTransformation.versionId, + transformationId: userTransformation.id, identifier: 'openfaas', testMode, }; diff --git a/src/util/customTransformer-v1.js b/src/util/customTransformer-v1.js index b1865dee6b..12dab547e6 100644 --- a/src/util/customTransformer-v1.js +++ b/src/util/customTransformer-v1.js @@ -65,7 +65,6 @@ async function userTransformHandlerV1( const isolatevmFactory = await getFactory( userTransformation.code, libraryVersionIds, - userTransformation.versionId, userTransformation.id, userTransformation.workspaceId, credentialsMap, diff --git a/src/util/customTransformer.js b/src/util/customTransformer.js index 4f4620fd2d..5ca1fae47c 100644 --- a/src/util/customTransformer.js +++ b/src/util/customTransformer.js @@ -16,9 +16,11 @@ async function runUserTransform( code, secrets, eventsMetadata, - versionId, + transformationId, + workspaceId, testMode = false, ) { + const trTags = { identifier: 'v0', transformationId, workspaceId }; // TODO: Decide on the right value for memory limit const isolate = new ivm.Isolate({ memoryLimit: ISOLATE_VM_MEMORY }); const context = await isolate.createContext(); @@ -36,9 +38,9 @@ async function runUserTransform( new ivm.Reference(async (resolve, ...args) => { try { const fetchStartTime = new Date(); - const res = await fetchWithDnsWrapper(versionId, ...args); + const res = await fetchWithDnsWrapper(trTags, ...args); const data = await res.json(); - stats.timing('fetch_call_duration', fetchStartTime, { versionId }); + stats.timing('fetch_call_duration', fetchStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(data).copyInto()]); } catch (error) { resolve.applyIgnored(undefined, [new ivm.ExternalCopy('ERROR').copyInto()]); @@ -51,7 +53,7 @@ async function runUserTransform( new ivm.Reference(async (resolve, reject, ...args) => { try { const fetchStartTime = new Date(); - const res = await fetchWithDnsWrapper(versionId, ...args); + const res = await fetchWithDnsWrapper(trTags, ...args); const headersContent = {}; res.headers.forEach((value, header) => { headersContent[header] = value; @@ -67,7 +69,7 @@ async function runUserTransform( data.body = JSON.parse(data.body); } catch (e) {} - stats.timing('fetchV2_call_duration', fetchStartTime, { versionId }); + stats.timing('fetchV2_call_duration', fetchStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(data).copyInto()]); } catch (error) { const err = JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error))); @@ -93,7 +95,7 @@ async function runUserTransform( throw new Error(`request to fetch geolocation failed with status code: ${res.status}`); } const geoData = await res.json(); - stats.timing('geo_call_duration', geoStartTime, { versionId }); + stats.timing('geo_call_duration', geoStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(geoData).copyInto()]); } catch (error) { const err = JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error))); @@ -251,12 +253,9 @@ async function runUserTransform( isolate.dispose(); const tags = { - identifier: 'v0', errored: transformationError ? true : false, ...(Object.keys(eventsMetadata).length ? getMetadata(Object.values(eventsMetadata)[0]) : {}), - ...(Object.keys(eventsMetadata).length - ? getTransformationMetadata(Object.values(eventsMetadata)[0]) - : {}), + ...trTags, }; stats.counter('user_transform_function_input_events', events.length, tags); @@ -318,7 +317,8 @@ async function userTransformHandler( res.code, res.secrets || {}, eventsMetadata, - versionId, + res.id, + res.workspaceId, testMode, ); diff --git a/src/util/customTransforrmationsStore-v1.js b/src/util/customTransforrmationsStore-v1.js index 6e2d799f3a..d2d14f318e 100644 --- a/src/util/customTransforrmationsStore-v1.js +++ b/src/util/customTransforrmationsStore-v1.js @@ -19,25 +19,19 @@ const getRudderLibrariesUrl = `${CONFIG_BACKEND_URL}/rudderstackTransformationLi async function getTransformationCodeV1(versionId) { const transformation = transformationCache[versionId]; if (transformation) return transformation; - const tags = { - versionId, - version: 1, - }; try { const url = `${getTransformationURL}?versionId=${versionId}`; - const startTime = new Date(); const response = await fetchWithProxy(url); responseStatusHandler(response.status, 'Transformation', versionId, url); - stats.increment('get_transformation_code', { success: 'true', ...tags }); - stats.timing('get_transformation_code_time', startTime, tags); - stats.timingSummary('get_transformation_code_time_summary', startTime, tags); const myJson = await response.json(); transformationCache[versionId] = myJson; return myJson; } catch (error) { - logger.error(error); - stats.increment('get_transformation_code', { success: 'false', ...tags }); + logger.error( + `Error fetching transformation V1 code for versionId: ${versionId}`, + error.message, + ); throw error; } } @@ -45,25 +39,16 @@ async function getTransformationCodeV1(versionId) { async function getLibraryCodeV1(versionId) { const library = libraryCache[versionId]; if (library) return library; - const tags = { - libraryVersionId: versionId, - version: 1, - }; try { const url = `${getLibrariesUrl}?versionId=${versionId}`; - const startTime = new Date(); const response = await fetchWithProxy(url); responseStatusHandler(response.status, 'Transformation Library', versionId, url); - stats.increment('get_libraries_code', { success: 'true', ...tags }); - stats.timing('get_libraries_code_time', startTime, tags); - stats.timingSummary('get_libraries_code_time_summary', startTime, tags); const myJson = await response.json(); libraryCache[versionId] = myJson; return myJson; } catch (error) { - logger.error(error); - stats.increment('get_libraries_code', { success: 'false', ...tags }); + logger.error(`Error fetching library code for versionId: ${versionId}`, error.message); throw error; } } @@ -71,27 +56,17 @@ async function getLibraryCodeV1(versionId) { async function getRudderLibByImportName(importName) { const rudderLibrary = rudderLibraryCache[importName]; if (rudderLibrary) return rudderLibrary; - const tags = { - libraryVersionId: importName, - version: 1, - type: 'rudderlibrary', - }; try { const [name, version] = importName.split('/').slice(-2); const url = `${getRudderLibrariesUrl}/${name}?version=${version}`; - const startTime = new Date(); const response = await fetchWithProxy(url); responseStatusHandler(response.status, 'Rudder Library', importName, url); - stats.increment('get_libraries_code', { success: 'true', ...tags }); - stats.timing('get_libraries_code_time', startTime, tags); - stats.timingSummary('get_libraries_code_time_summary', startTime, tags); const myJson = await response.json(); rudderLibraryCache[importName] = myJson; return myJson; } catch (error) { - logger.error(error); - stats.increment('get_libraries_code', { success: 'false', ...tags }); + logger.error(`Error fetching rudder library code for importName: ${importName}`, error.message); throw error; } } diff --git a/src/util/customTransforrmationsStore.js b/src/util/customTransforrmationsStore.js index 2c5a7b446d..2043d18875 100644 --- a/src/util/customTransforrmationsStore.js +++ b/src/util/customTransforrmationsStore.js @@ -2,7 +2,6 @@ const NodeCache = require('node-cache'); const { fetchWithProxy } = require('./fetch'); const logger = require('../logger'); const { responseStatusHandler } = require('./utils'); -const stats = require('./stats'); const myCache = new NodeCache({ stdTTL: 60 * 60 * 24 * 1 }); @@ -18,19 +17,14 @@ async function getTransformationCode(versionId) { if (transformation) return transformation; try { const url = `${getTransformationURL}?versionId=${versionId}`; - const startTime = new Date(); const response = await fetchWithProxy(url); responseStatusHandler(response.status, 'Transformation', versionId, url); - stats.increment('get_transformation_code', { versionId, success: 'true' }); - stats.timing('get_transformation_code_time', startTime, { versionId }); - stats.timingSummary('get_transformation_code_time_summary', startTime, { versionId }); const myJson = await response.json(); myCache.set(versionId, myJson); return myJson; } catch (error) { - logger.error(error); - stats.increment('get_transformation_code', { versionId, success: 'false' }); + logger.error(`Error fetching transformation code for versionId: ${versionId}`, error.message); throw error; } } diff --git a/src/util/ivmFactory.js b/src/util/ivmFactory.js index 44beff01dc..625591964c 100644 --- a/src/util/ivmFactory.js +++ b/src/util/ivmFactory.js @@ -33,13 +33,13 @@ async function loadModule(isolateInternal, contextInternal, moduleName, moduleCo async function createIvm( code, libraryVersionIds, - versionId, transformationId, workspaceId, credentials, secrets, testMode, ) { + const trTags = { identifier: 'V1', transformationId, workspaceId }; const createIvmStartTime = new Date(); const logs = []; const libraries = await Promise.all( @@ -187,9 +187,9 @@ async function createIvm( new ivm.Reference(async (resolve, ...args) => { try { const fetchStartTime = new Date(); - const res = await fetchWithDnsWrapper(versionId, ...args); + const res = await fetchWithDnsWrapper(trTags, ...args); const data = await res.json(); - stats.timing('fetch_call_duration', fetchStartTime, { versionId }); + stats.timing('fetch_call_duration', fetchStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(data).copyInto()]); } catch (error) { resolve.applyIgnored(undefined, [new ivm.ExternalCopy('ERROR').copyInto()]); @@ -202,7 +202,7 @@ async function createIvm( new ivm.Reference(async (resolve, reject, ...args) => { try { const fetchStartTime = new Date(); - const res = await fetchWithDnsWrapper(versionId, ...args); + const res = await fetchWithDnsWrapper(trTags, ...args); const headersContent = {}; res.headers.forEach((value, header) => { headersContent[header] = value; @@ -218,7 +218,7 @@ async function createIvm( data.body = JSON.parse(data.body); } catch (e) {} - stats.timing('fetchV2_call_duration', fetchStartTime, { versionId }); + stats.timing('fetchV2_call_duration', fetchStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(data).copyInto()]); } catch (error) { const err = JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error))); @@ -243,7 +243,7 @@ async function createIvm( throw new Error(`request to fetch geolocation failed with status code: ${res.status}`); } const geoData = await res.json(); - stats.timing('geo_call_duration', geoStartTime, { versionId }); + stats.timing('geo_call_duration', geoStartTime, trTags); resolve.applyIgnored(undefined, [new ivm.ExternalCopy(geoData).copyInto()]); } catch (error) { const err = JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error))); @@ -257,7 +257,7 @@ async function createIvm( logger.error( `Error fetching credentials map for transformationID: ${transformationId} and workspaceId: ${workspaceId}`, ); - stats.increment('credential_error_total', { transformationId, workspaceId }); + stats.increment('credential_error_total', trTags); return undefined; } if (key === null || key === undefined) { @@ -416,7 +416,7 @@ async function createIvm( reference: true, }); const fName = availableFuncNames[0]; - stats.timing('createivm_duration', createIvmStartTime); + stats.timing('createivm_duration', createIvmStartTime, trTags); // TODO : check if we can resolve this // eslint-disable-next-line no-async-promise-executor @@ -446,7 +446,6 @@ async function getFactory( libraryVersionIds, transformationId, workspaceId, - versionId, credentials, secrets, testMode, @@ -456,7 +455,6 @@ async function getFactory( return createIvm( code, libraryVersionIds, - versionId, transformationId, workspaceId, credentials, diff --git a/src/util/prometheus.js b/src/util/prometheus.js index ffcfda3784..860c266565 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -239,79 +239,12 @@ class Prometheus { 'implementation', ], }, - { - name: 'tp_violation_type', - help: 'tp_violation_type', - type: 'counter', - labelNames: ['violationType', 'sourceType', 'destinationType', 'k8_namespace'], - }, - { - name: 'tp_propagated_events', - help: 'tp_propagated_events', - type: 'counter', - labelNames: ['sourceType', 'destinationType', 'k8_namespace'], - }, - { - name: 'tp_errors', - help: 'tp_errors', - type: 'counter', - labelNames: [ - 'sourceType', - 'destinationType', - 'k8_namespace', - 'workspaceId', - 'trackingPlanId', - ], - }, - { - name: 'tp_events_count', - help: 'tp_events_count', - type: 'counter', - labelNames: ['sourceType', 'destinationType', 'k8_namespace'], - }, - { - name: 'user_transform_function_group_size', - help: 'user_transform_function_group_size', - type: 'counter', - labelNames: ['processSessions'], - }, - { - name: 'user_transform_errors', - help: 'user_transform_errors', - type: 'counter', - labelNames: [ - 'workspaceId', - 'transformationId', - 'status', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, - { - name: 'c2', - help: 'h2', - type: 'counter', - labelNames: [ - 'transformationVersionId', - 'processSessions', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, { name: 'dest_transform_requests', help: 'dest_transform_requests', type: 'counter', labelNames: ['destination', 'version', 'sourceType', 'destinationType', 'k8_namespace'], }, - { - name: 'user_transform_requests', - help: 'user_transform_requests', - type: 'counter', - labelNames: ['processSessions'], - }, { name: 'source_transform_requests', help: 'source_transform_requests', @@ -342,56 +275,6 @@ class Prometheus { type: 'counter', labelNames: ['success'], }, - { - name: 'create_zip_error', - help: 'create_zip_error', - type: 'counter', - labelNames: ['fileName'], - }, - { - name: 'delete_zip_error', - help: 'delete_zip_error', - type: 'counter', - labelNames: ['functionName'], - }, - { - name: 'hv_metrics', - help: 'hv_metrics', - type: 'counter', - labelNames: [ - 'destination', - 'version', - 'sourceType', - 'destinationType', - 'k8_namespace', - 'dropped', - 'violationType', - ], - }, - { - name: 'events_into_vm', - help: 'events_into_vm', - type: 'counter', - labelNames: [ - 'transformerVersionId', - 'version', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, - { - name: 'missing_handle', - help: 'missing_handle', - type: 'counter', - labelNames: [ - 'transformerVersionId', - 'language', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, { name: 'proxy_test_error', help: 'proxy_test_error', @@ -495,18 +378,6 @@ class Prometheus { type: 'counter', labelNames: ['destinationId'], }, - { - name: 'get_eventSchema_error', - help: 'get_eventSchema_error', - type: 'counter', - labelNames: [], - }, - { - name: 'get_tracking_plan_error', - help: 'get_tracking_plan_error', - type: 'counter', - labelNames: [], - }, { name: 'redis_error', help: 'redis_error', @@ -537,18 +408,6 @@ class Prometheus { type: 'counter', labelNames: ['writeKey', 'source'], }, - { - name: 'get_transformation_code', - help: 'get_transformation_code', - type: 'counter', - labelNames: ['versionId', 'version', 'success'], - }, - { - name: 'get_libraries_code', - help: 'get_libraries_code', - type: 'counter', - labelNames: ['libraryVersionId', 'version', 'type', 'success'], - }, { name: 'invalid_shopify_event', help: 'invalid_shopify_event', @@ -574,12 +433,6 @@ class Prometheus { 'sourceId', ], }, - { - name: 'credential_error_total', - help: 'Error in fetching credentials count', - type: 'counter', - labelNames: ['transformationId', 'workspaceId'], - }, // Gauges { @@ -667,118 +520,18 @@ class Prometheus { type: 'histogram', labelNames: ['method', 'route', 'code', 'destType'], }, - { - name: 'tp_batch_size', - help: 'Size of batch of events for tracking plan validation', - type: 'histogram', - buckets: [ - 1024, 102400, 524288, 1048576, 10485760, 20971520, 52428800, 104857600, 209715200, - 524288000, - ], - labelNames: [ - 'sourceType', - 'destinationType', - 'k8_namespace', - 'workspaceId', - 'trackingPlanId', - ], - }, - { - name: 'tp_event_validation_latency', - help: 'Latency of validating tracking plan at event level', - type: 'histogram', - labelNames: [ - 'sourceType', - 'destinationType', - 'k8_namespace', - 'workspaceId', - 'trackingPlanId', - 'status', - 'exception', - ], - }, - { - name: 'tp_batch_validation_latency', - help: 'Latency of validating tracking plan at batch level', - type: 'histogram', - labelNames: [ - 'sourceType', - 'destinationType', - 'k8_namespace', - 'workspaceId', - 'trackingPlanId', - ], - }, { name: 'cdk_events_latency', help: 'cdk_events_latency', type: 'histogram', labelNames: ['destination', 'sourceType', 'destinationType', 'k8_namespace'], }, - { - name: 'tp_event_latency', - help: 'tp_event_latency', - type: 'histogram', - labelNames: ['sourceType', 'destinationType', 'k8_namespace'], - }, { name: 'regulation_worker_requests_dest_latency', help: 'regulation_worker_requests_dest_latency', type: 'histogram', labelNames: ['feature', 'implementation', 'destType'], }, - { - name: 'user_transform_request_latency', - help: 'user_transform_request_latency', - type: 'histogram', - labelNames: [ - 'workspaceId', - 'transformationId', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, - { - name: 'user_transform_request_latency_summary', - help: 'user_transform_request_latency_summary', - type: 'summary', - labelNames: [ - 'workspaceId', - 'transformationId', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, - { - name: 'user_transform_batch_size', - help: 'user_transform_batch_size', - type: 'histogram', - labelNames: [ - 'workspaceId', - 'transformationId', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - buckets: [ - 1024, 102400, 524288, 1048576, 10485760, 20971520, 52428800, 104857600, 209715200, - 524288000, - ], // 1KB, 100KB, 0.5MB, 1MB, 10MB, 20MB, 50MB, 100MB, 200MB, 500MB - }, - { - name: 'user_transform_batch_size_summary', - help: 'user_transform_batch_size_summary', - type: 'summary', - labelNames: [ - 'workspaceId', - 'transformationId', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, { name: 'source_transform_request_latency', help: 'source_transform_request_latency', @@ -797,88 +550,6 @@ class Prometheus { type: 'histogram', labelNames: ['destination', 'version'], }, - { - name: 'creation_time', - help: 'creation_time', - type: 'histogram', - labelNames: ['transformerVersionId', 'language', 'identifier', 'publish', 'testMode'], - }, - { name: 'get_tracking_plan', help: 'get_tracking_plan', type: 'histogram', labelNames: [] }, - { name: 'createivm_duration', help: 'createivm_duration', type: 'histogram', labelNames: [] }, - { - name: 'fetchV2_call_duration', - help: 'fetchV2_call_duration', - type: 'histogram', - labelNames: ['versionId'], - }, - { - name: 'fetch_call_duration', - help: 'fetch_call_duration', - type: 'histogram', - labelNames: ['versionId'], - }, - { - name: 'fetch_dns_resolve_time', - help: 'fetch_dns_resolve_time', - type: 'histogram', - labelNames: ['transformerVersionId', 'error'], - }, - { - name: 'geo_call_duration', - help: 'geo_call_duration', - type: 'histogram', - labelNames: ['versionId'], - }, - { - name: 'get_transformation_code_time', - help: 'get_transformation_code_time', - type: 'histogram', - labelNames: ['versionId', 'version'], - }, - { - name: 'get_transformation_code_time_summary', - help: 'get_transformation_code_time_summary', - type: 'summary', - labelNames: ['versionId', 'version'], - }, - { - name: 'get_libraries_code_time', - help: 'get_libraries_code_time', - type: 'histogram', - labelNames: ['libraryVersionId', 'versionId', 'type', 'version'], - }, - { - name: 'get_libraries_code_time_summary', - help: 'get_libraries_code_time_summary', - type: 'summary', - labelNames: ['libraryVersionId', 'versionId', 'type', 'version'], - }, - { - name: 'isolate_cpu_time', - help: 'isolate_cpu_time', - type: 'histogram', - labelNames: [ - 'transformerVersionId', - 'identifier', - 'version', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, - { - name: 'isolate_wall_time', - help: 'isolate_wall_time', - type: 'histogram', - labelNames: [ - 'transformerVersionId', - 'identifier', - 'version', - 'sourceType', - 'destinationType', - 'k8_namespace', - ], - }, { name: 'marketo_bulk_upload_process_time', help: 'marketo_bulk_upload_process_time', @@ -1004,20 +675,6 @@ class Prometheus { labelNames: ['destination', 'version', 'sourceType', 'destinationType', 'k8_namespace'], buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], }, - { - name: 'user_transform_input_events', - help: 'Number of input events to user transform', - type: 'histogram', - labelNames: ['processSessions'], - buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], - }, - { - name: 'user_transform_output_events', - help: 'user_transform_output_events', - type: 'histogram', - labelNames: ['processSessions'], - buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], - }, { name: 'marketo_bulk_upload_create_header_time', help: 'marketo_bulk_upload_create_header_time', @@ -1054,6 +711,117 @@ class Prometheus { type: 'histogram', labelNames: [], }, + // tracking plan metrics: + // counter + { + name: 'hv_metrics', + help: 'hv_metrics', + type: 'counter', + labelNames: [ + 'destination', + 'version', + 'sourceType', + 'destinationType', + 'k8_namespace', + 'dropped', + 'violationType', + ], + }, + { + name: 'get_eventSchema_error', + help: 'get_eventSchema_error', + type: 'counter', + labelNames: [], + }, + { + name: 'get_tracking_plan_error', + help: 'get_tracking_plan_error', + type: 'counter', + labelNames: [], + }, + // histogram + { + name: 'tp_batch_size', + help: 'Size of batch of events for tracking plan validation', + type: 'histogram', + buckets: [ + 1024, 102400, 524288, 1048576, 10485760, 20971520, 52428800, 104857600, 209715200, + 524288000, + ], + labelNames: [ + 'sourceType', + 'destinationType', + 'k8_namespace', + 'workspaceId', + 'trackingPlanId', + ], + }, + { + name: 'tp_event_validation_latency', + help: 'Latency of validating tracking plan at event level', + type: 'histogram', + labelNames: [ + 'sourceType', + 'destinationType', + 'k8_namespace', + 'workspaceId', + 'trackingPlanId', + 'status', + 'exception', + ], + }, + { + name: 'tp_batch_validation_latency', + help: 'Latency of validating tracking plan at batch level', + type: 'histogram', + labelNames: [ + 'sourceType', + 'destinationType', + 'k8_namespace', + 'workspaceId', + 'trackingPlanId', + ], + }, + { + name: 'tp_event_latency', + help: 'tp_event_latency', + type: 'histogram', + labelNames: ['sourceType', 'destinationType', 'k8_namespace'], + }, + { name: 'get_tracking_plan', help: 'get_tracking_plan', type: 'histogram', labelNames: [] }, + // User transform metrics + // counter + { + name: 'user_transform_function_group_size', + help: 'user_transform_function_group_size', + type: 'counter', + labelNames: [], + }, + { + name: 'user_transform_errors', + help: 'user_transform_errors', + type: 'counter', + labelNames: [ + 'workspaceId', + 'transformationId', + 'status', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + }, + { + name: 'user_transform_requests', + help: 'user_transform_requests', + type: 'counter', + labelNames: [], + }, + { + name: 'credential_error_total', + help: 'Error in fetching credentials count', + type: 'counter', + labelNames: ['identifier', 'transformationId', 'workspaceId'], + }, { name: 'user_transform_function_input_events', help: 'user_transform_function_input_events', @@ -1070,6 +838,85 @@ class Prometheus { 'workspaceId', ], }, + // histogram + { + name: 'user_transform_request_latency', + help: 'user_transform_request_latency', + type: 'histogram', + labelNames: [ + 'workspaceId', + 'transformationId', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + }, + { + name: 'user_transform_batch_size', + help: 'user_transform_batch_size', + type: 'histogram', + labelNames: [ + 'workspaceId', + 'transformationId', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + buckets: [ + 1024, 102400, 524288, 1048576, 10485760, 20971520, 52428800, 104857600, 209715200, + 524288000, + ], // 1KB, 100KB, 0.5MB, 1MB, 10MB, 20MB, 50MB, 100MB, 200MB, 500MB + }, + { + name: 'creation_time', + help: 'creation_time', + type: 'histogram', + labelNames: ['transformationId', 'identifier', 'testMode'], + }, + { + name: 'createivm_duration', + help: 'createivm_duration', + type: 'histogram', + labelNames: ['identifier', 'transformationId', 'workspaceId'], + }, + { + name: 'fetchV2_call_duration', + help: 'fetchV2_call_duration', + type: 'histogram', + labelNames: ['identifier', 'transformationId', 'workspaceId'], + }, + { + name: 'fetch_call_duration', + help: 'fetch_call_duration', + type: 'histogram', + labelNames: ['identifier', 'transformationId', 'workspaceId'], + }, + { + name: 'fetch_dns_resolve_time', + help: 'fetch_dns_resolve_time', + type: 'histogram', + labelNames: ['identifier', 'transformationId', 'workspaceId', 'error'], + }, + { + name: 'geo_call_duration', + help: 'geo_call_duration', + type: 'histogram', + labelNames: ['identifier', 'transformationId', 'workspaceId'], + }, + { + name: 'user_transform_input_events', + help: 'Number of input events to user transform', + type: 'histogram', + labelNames: [], + buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], + }, + { + name: 'user_transform_output_events', + help: 'user_transform_output_events', + type: 'histogram', + labelNames: [], + buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], + }, { name: 'user_transform_function_latency', help: 'user_transform_function_latency', @@ -1086,6 +933,31 @@ class Prometheus { 'workspaceId', ], }, + // summary + { + name: 'user_transform_request_latency_summary', + help: 'user_transform_request_latency_summary', + type: 'summary', + labelNames: [ + 'workspaceId', + 'transformationId', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + }, + { + name: 'user_transform_batch_size_summary', + help: 'user_transform_batch_size_summary', + type: 'summary', + labelNames: [ + 'workspaceId', + 'transformationId', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + }, { name: 'user_transform_function_latency_summary', help: 'user_transform_function_latency_summary', diff --git a/src/util/utils.js b/src/util/utils.js index 1ac70b9541..eb5a011444 100644 --- a/src/util/utils.js +++ b/src/util/utils.js @@ -16,7 +16,7 @@ const LOCAL_HOST_NAMES_LIST = ['localhost', '127.0.0.1', '[::]', '[::1]']; const LOCALHOST_OCTET = '127.'; const RECORD_TYPE_A = 4; // ipv4 -const staticLookup = (transformerVersionId) => async (hostname, _, cb) => { +const staticLookup = (transformationTags) => async (hostname, _, cb) => { let ips; const resolveStartTime = new Date(); try { @@ -24,13 +24,13 @@ const staticLookup = (transformerVersionId) => async (hostname, _, cb) => { } catch (error) { logger.error(`DNS Error Code: ${error.code} | Message : ${error.message}`); stats.timing('fetch_dns_resolve_time', resolveStartTime, { - transformerVersionId, + ...transformationTags, error: 'true', }); cb(null, `unable to resolve IP address for ${hostname}`, RECORD_TYPE_A); return; } - stats.timing('fetch_dns_resolve_time', resolveStartTime, { transformerVersionId }); + stats.timing('fetch_dns_resolve_time', resolveStartTime, transformationTags); if (ips.length === 0) { cb(null, `resolved empty list of IP address for ${hostname}`, RECORD_TYPE_A); @@ -48,9 +48,9 @@ const staticLookup = (transformerVersionId) => async (hostname, _, cb) => { cb(null, ips[0], RECORD_TYPE_A); }; -const httpAgentWithDnsLookup = (scheme, transformerVersionId) => { +const httpAgentWithDnsLookup = (scheme, transformationTags) => { const httpModule = scheme === 'http' ? http : https; - return new httpModule.Agent({ lookup: staticLookup(transformerVersionId) }); + return new httpModule.Agent({ lookup: staticLookup(transformationTags) }); }; const blockLocalhostRequests = (url) => { @@ -74,7 +74,7 @@ const blockInvalidProtocolRequests = (url) => { } }; -const fetchWithDnsWrapper = async (transformerVersionId, ...args) => { +const fetchWithDnsWrapper = async (transformationTags, ...args) => { if (process.env.DNS_RESOLVE_FETCH_HOST !== 'true') { return await fetch(...args); } @@ -88,7 +88,7 @@ const fetchWithDnsWrapper = async (transformerVersionId, ...args) => { const fetchOptions = args[1] || {}; const schemeName = fetchURL.startsWith('https') ? 'https' : 'http'; // assign resolved agent to fetch - fetchOptions.agent = httpAgentWithDnsLookup(schemeName, transformerVersionId); + fetchOptions.agent = httpAgentWithDnsLookup(schemeName, transformationTags); return await fetch(fetchURL, fetchOptions); };