chore: adding a metric to capture event batch size DAT-909 (#3070)

* chore: adding a metric to capture event batch size
rudderlabs · Mar 15, 2024 · 5befa02 · 5befa02
1 parent f00d411
commit 5befa02
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 6 deletions.
diff --git a/src/controllers/userTransform.ts b/src/controllers/userTransform.ts
@@ -15,9 +15,10 @@ export class UserTransformController {
       '(User transform - router:/customTransform ):: Request to transformer',
       JSON.stringify(ctx.request.body),
     );
+    const requestSize = Number(ctx.request.get('content-length'));
     const events = ctx.request.body as ProcessorTransformationRequest[];
     const processedRespone: UserTransformationServiceResponse =
-      await UserTransformService.transformRoutine(events, ctx.state.features);
+      await UserTransformService.transformRoutine(events, ctx.state.features, requestSize);
     ctx.body = processedRespone.transformedEvents;
     ControllerUtility.postProcess(ctx, processedRespone.retryStatus);
     logger.debug(

diff --git a/src/services/userTransform.ts b/src/services/userTransform.ts
@@ -14,7 +14,7 @@ import {
   RetryRequestError,
   extractStackTraceUptoLastSubstringMatch,
 } from '../util/utils';
-import { getMetadata, isNonFuncObject } from '../v0/util';
+import { getMetadata, getTransformationMetadata, isNonFuncObject } from '../v0/util';
 import { SUPPORTED_FUNC_NAMES } from '../util/ivmFactory';
 import logger from '../logger';
 import stats from '../util/stats';
@@ -28,6 +28,7 @@ export class UserTransformService {
   public static async transformRoutine(
     events: ProcessorTransformationRequest[],
     features: FeatureFlags = {},
+    requestSize = 0,
   ): Promise<UserTransformationServiceResponse> {
     let retryStatus = 200;
     const groupedEvents: NonNullable<unknown> = groupBy(
@@ -162,16 +163,19 @@ export class UserTransformService {
             ),
           );
           stats.counter('user_transform_errors', eventsToProcess.length, {
-            transformationId: eventsToProcess[0]?.metadata?.transformationId,
-            workspaceId: eventsToProcess[0]?.metadata?.workspaceId,
             status,
             ...metaTags,
+            ...getTransformationMetadata(eventsToProcess[0]?.metadata),
           });
         } finally {
           stats.timing('user_transform_request_latency', userFuncStartTime, {
-            workspaceId: eventsToProcess[0]?.metadata?.workspaceId,
-            transformationId: eventsToProcess[0]?.metadata?.transformationId,
             ...metaTags,
+            ...getTransformationMetadata(eventsToProcess[0]?.metadata),
+          });
+
+          stats.histogram('user_transform_batch_size', requestSize, {
+            ...metaTags,
+            ...getTransformationMetadata(eventsToProcess[0]?.metadata),
           });
         }
 

diff --git a/src/util/prometheus.js b/src/util/prometheus.js
@@ -593,6 +593,10 @@ class Prometheus {
         name: 'tp_batch_size',
         help: 'Size of batch of events for tracking plan validation',
         type: 'histogram',
+        buckets: [
+          1024, 102400, 524288, 1048576, 10485760, 20971520, 52428800, 104857600, 209715200,
+          524288000,
+        ],
         labelNames: [
           'sourceType',
           'destinationType',
@@ -670,6 +674,22 @@ class Prometheus {
           'k8_namespace',
         ],
       },
+      {
+        name: 'user_transform_batch_size',
+        help: 'user_transform_batch_size',
+        type: 'histogram',
+        labelNames: [
+          'workspaceId',
+          'transformationId',
+          'sourceType',
+          'destinationType',
+          'k8_namespace',
+        ],
+        buckets: [
+          1024, 102400, 524288, 1048576, 10485760, 20971520, 52428800, 104857600, 209715200,
+          524288000,
+        ], // 1KB, 100KB, 0.5MB, 1MB, 10MB, 20MB, 50MB, 100MB, 200MB, 500MB
+      },
       {
         name: 'source_transform_request_latency',
         help: 'source_transform_request_latency',