chore(release): pull release/v1.31.0 into main (#2237)

CHANGELOG.md

@@ -2,6 +2,25 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.31.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.30.1...v1.31.0) (2023-06-05)
+
+
+### Features
+
+* add stats for bad events in fb_custom_audience ([#2192](https://github.com/rudderlabs/rudder-transformer/issues/2192)) ([8176874](https://github.com/rudderlabs/rudder-transformer/commit/8176874bd599c999a70a382577c7605a1d3b3695))
+* error enrichment to get clear error messages for facebook pixel ([#2018](https://github.com/rudderlabs/rudder-transformer/issues/2018)) ([0ee7c73](https://github.com/rudderlabs/rudder-transformer/commit/0ee7c7348e363ec542c5e8b63d43854f8c8672c2)), closes [#2233](https://github.com/rudderlabs/rudder-transformer/issues/2233)
+
+
+### Bug Fixes
+
+* **cdkv1:** error handling for cdkv1 destinations ([#2227](https://github.com/rudderlabs/rudder-transformer/issues/2227)) ([c3aedbf](https://github.com/rudderlabs/rudder-transformer/commit/c3aedbfe41c176c9791770cc52b1a4cb1dddbefe))
+* **customerio destination:** added fix for unhandled exception for pr… ([#2226](https://github.com/rudderlabs/rudder-transformer/issues/2226)) ([fde530f](https://github.com/rudderlabs/rudder-transformer/commit/fde530f542b4229220f93c09a5ac0bd2ef563507))
+* **hubspot  destination:** added fix for handling error when  hubspotPropertyMapResponse not present ([d1613b6](https://github.com/rudderlabs/rudder-transformer/commit/d1613b6ae275d7e884b21eccbf2a9391040d3ef1))
+* hubspot bugsnag error ([#2235](https://github.com/rudderlabs/rudder-transformer/issues/2235)) ([1d4df3a](https://github.com/rudderlabs/rudder-transformer/commit/1d4df3ab8f7deb89eaaec154500bd4c94350c551))
+* **hubspot destination:** add fix for handling  error case when properties are not sent in hubspot ([0225a28](https://github.com/rudderlabs/rudder-transformer/commit/0225a280f1efd051d2d732d633d1cbab7b0f37ea))
+* iterable bugsnag error ([#2234](https://github.com/rudderlabs/rudder-transformer/issues/2234)) ([b2b3b19](https://github.com/rudderlabs/rudder-transformer/commit/b2b3b1973ddd2e298befdcc30ac2b1be44023736))
+* sonarCloud code smells ([#2187](https://github.com/rudderlabs/rudder-transformer/issues/2187)) ([6646257](https://github.com/rudderlabs/rudder-transformer/commit/66462570823a08c21d55a519f59fbf19af0a54b9))
+
 ### [1.30.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.30.0...v1.30.1) (2023-05-31)
 
 ## [1.30.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.29.2...v1.30.0) (2023-05-30)

CODEOWNERS

@@ -1,8 +1,9 @@
-* @rudderlabs/integrations-platform @sivashanmukh @lvrach
-__tests__/ @rudderlabs/integrations
+* @rudderlabs/integrations-platform @sivashanmukh
+__tests__/ @rudderlabs/integrations @rudderlabs/data-management
 __mocks__/ @rudderlabs/integrations
 v0/ @rudderlabs/integrations
 cdk/ @rudderlabs/integrations
 src/features.json @rudderlabs/integrations
 constants/ @rudderlabs/integrations
 warehouse/ @rudderlabs/warehouse
+src/util/ @rudderlabs/data-management

jest.config.js

@@ -63,6 +63,7 @@ module.exports = {
   //     diagnostics: false
   //   },
   // },
+  preset: 'ts-jest/presets/js-with-ts-esm',
 
   // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
   // maxWorkers: "50%",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.30.1",
+  "version": "1.31.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/cdk/v1/handler.js

@@ -81,4 +81,5 @@ async function processCdkV1(destType, parsedEvent) {
 
 module.exports = {
   processCdkV1,
+  getErrorInfo,
 };

src/middleware.js

@@ -25,11 +25,11 @@ function requestSizeMiddleware() {
       route: ctx.request.url,
     };
 
-    const inputLength =
-      ctx.request && ctx.request.body ? Buffer.byteLength(JSON.stringify(ctx.request.body)) : 0;
+    const inputLength = ctx.request?.body ? Buffer.byteLength(JSON.stringify(ctx.request.body)) : 0;
     stats.histogram('http_request_size', inputLength, labels);
-    const outputLength =
-      ctx.response && ctx.response.body ? Buffer.byteLength(JSON.stringify(ctx.response.body)) : 0;
+    const outputLength = ctx.response?.body
+      ? Buffer.byteLength(JSON.stringify(ctx.response.body))
+      : 0;
     stats.histogram('http_response_size', outputLength, labels);
   };
 }

src/services/destination/cdkV1Integration.ts

@@ -1,4 +1,4 @@
-import { ConfigFactory, Executor } from 'rudder-transformer-cdk';
+import { ConfigFactory, Executor, RudderBaseConfig } from 'rudder-transformer-cdk';
 import path from 'path';
 import IntegrationDestinationService from '../../interfaces/DestinationService';
 import {
@@ -16,6 +16,7 @@ import {
 import { TransformationError } from '../../v0/util/errorTypes';
 import DestinationPostTransformationService from './postTransformation';
 import tags from '../../v0/util/tags';
+import { getErrorInfo } from '../../cdk/v1/handler';
 
 export default class CDKV1DestinationService implements IntegrationDestinationService {
   public init() {
@@ -49,6 +50,15 @@ export default class CDKV1DestinationService implements IntegrationDestinationSe
     return metaTO;
   }
 
+  private async processCDKV1(event: any, tfConfig: RudderBaseConfig): Promise<any> {
+    try {
+      const respEvents = await Executor.execute(event, tfConfig);
+      return respEvents;
+    } catch (error) {
+      throw getErrorInfo(error);
+    }
+  }
+
   public async doProcessorTransformation(
     events: ProcessorTransformationRequest[],
     destinationType: string,
@@ -59,7 +69,7 @@ export default class CDKV1DestinationService implements IntegrationDestinationSe
     const respList: ProcessorTransformationResponse[][] = await Promise.all(
       events.map(async (event) => {
         try {
-          const transformedPayloads: any = await Executor.execute(event as any, tfConfig);
+          const transformedPayloads: any = await this.processCDKV1(event as any, tfConfig);
           // We are not passing destinationHandler to post processor as we don't have post processing in CDK flows
           return DestinationPostTransformationService.handleProcessorTransformSucessEvents(
             event,

src/util/error-extractor/index.ts

@@ -0,0 +1,78 @@
+/* eslint-disable max-classes-per-file */
+import { MessageDetails, StatusCode } from "./types";
+
+export class ErrorDetailsExtractor {
+  status: StatusCode;
+
+  messageDetails: MessageDetails;
+
+  constructor (builder: ErrorDetailsExtractorBuilder) {
+    this.status = builder.getStatus();
+    this.messageDetails = builder.getMessageDetails();
+  }
+
+}
+
+export class ErrorDetailsExtractorBuilder {
+  status: StatusCode;
+
+  messageDetails: MessageDetails;
+
+  constructor() {
+    this.status = 0;
+    this.messageDetails = {};
+  }
+
+  setStatus(status: number): ErrorDetailsExtractorBuilder {
+    this.status = status;
+    return this;
+  }
+
+  /**
+   * This means we need to set a message from a specific field that we see from the destination's response
+   * 
+   * @param {string} fieldPath -- Path of the field which should be set as "error message"
+   * @returns 
+   */
+  setMessageField(fieldPath: string): ErrorDetailsExtractorBuilder {
+    if (this.messageDetails?.message) {
+      // This check basically ensures that "setMessage" was not already before
+      return this;
+    }
+    this.messageDetails = {
+      field: fieldPath
+    }
+    return this;
+  }
+
+  /**
+   * This means we need to set the message provided
+   * 
+   * @param {string} msg - error message
+   * @returns 
+   */
+  setMessage(msg: string): ErrorDetailsExtractorBuilder {
+    if (this.messageDetails?.field) {
+      // This check basically ensures that "setMessageField" was not already called before
+      return this;
+    }
+    this.messageDetails = {
+      message: msg
+    }
+    return this;
+  }
+
+  build(): ErrorDetailsExtractor {
+    return new ErrorDetailsExtractor(this)
+  }
+
+  getStatus(): number {
+    return this.status;
+  }
+
+  getMessageDetails(): Record<string, string> {
+    return this.messageDetails;
+  }
+}
+
+

src/util/error-extractor/types.ts

@@ -0,0 +1,2 @@
+export type MessageDetails = Record<string, string>;
+export type StatusCode = number;

src/util/eventValidation.js

@@ -244,69 +244,83 @@ async function validate(event) {
   }
 }
 
-function handleValidationErrors(validationErrors, mergedTpConfig, curDropEvent, curViolationType) {
+function handleValidationErrors(validationErrors, metadata, curDropEvent, curViolationType) {
   let dropEvent = curDropEvent;
   let violationType = curViolationType;
+  const {
+    mergedTpConfig,
+    destinationId = 'Non-determininable',
+    destinationType = 'Non-determininable',
+  } = metadata;
   const violationsByType = new Set(validationErrors.map((err) => err.type));
-  // eslint-disable-next-line no-restricted-syntax
-  for (const [key, val] of Object.entries(mergedTpConfig)) {
+
+  const handleUnknownOption = (value, key) => {
+    logger.error(
+      `Unknown option ${value} in ${key} for destId ${destinationId}, destType ${destinationType}`,
+    );
+  };
+
+  const handleAllowUnplannedEvents = (value) => {
+    if (!['true', 'false'].includes(value)) {
+      handleUnknownOption(value, 'allowUnplannedEvents');
+    } else if (value === 'false' && violationsByType.has(violationTypes.UnplannedEvent)) {
+      dropEvent = true;
+      violationType = violationTypes.UnplannedEvent;
+    }
+  };
+
+  const handleUnplannedProperties = (value) => {
+    const exists = violationsByType.has(violationTypes.AdditionalProperties);
+    if (!['forward', 'drop'].includes(value)) {
+      handleUnknownOption(value, 'unplannedProperties');
+    } else if (value === 'drop' && exists) {
+      dropEvent = true;
+      violationType = violationTypes.AdditionalProperties;
+    }
+  };
+
+  const handleAnyOtherViolation = (value) => {
+    if (!['forward', 'drop'].includes(value)) {
+      handleUnknownOption(value, 'anyOtherViolation');
+      return;
+    }
+
+    const violationTypesToCheck = [
+      violationTypes.UnknownViolation,
+      violationTypes.DatatypeMismatch,
+      violationTypes.RequiredMissing,
+    ];
+
+    const existingViolationType = violationTypesToCheck.find((type) => violationsByType.has(type));
+
+    if (value === 'drop' && existingViolationType) {
+      dropEvent = true;
+      violationType = existingViolationType;
+    }
+  };
+
+  const handleSendViolatedEventsTo = (value) => {
+    if (value !== 'procerrors') {
+      handleUnknownOption(value, 'sendViolatedEventsTo');
+    }
+  };
+
+  const handlerMap = {
+    allowUnplannedEvents: handleAllowUnplannedEvents,
+    unplannedProperties: handleUnplannedProperties,
+    anyOtherViolation: handleAnyOtherViolation,
+    sendViolatedEventsTo: handleSendViolatedEventsTo,
+  };
+
+  const foundConfig = Object.keys(mergedTpConfig).find((key) => handlerMap.hasOwnProperty(key));
+  if (foundConfig) {
     // To have compatibility for config-backend, spread-sheet plugin and postman collection
     // We are making everything to lower string and doing string comparison.
-    const value = val?.toString()?.toLowerCase();
-    // eslint-disable-next-line default-case
-    switch (key) {
-      case 'allowUnplannedEvents': {
-        const exists = violationsByType.has(violationTypes.UnplannedEvent);
-        if (value === 'false' && exists) {
-          dropEvent = true;
-          violationType = violationTypes.UnplannedEvent;
-          break;
-        }
-        if (!(value === 'true' || value === 'false')) {
-          logger.error(`Unknown option ${value} in ${key}"`);
-        }
-        break;
-      }
-      case 'unplannedProperties': {
-        const exists = violationsByType.has(violationTypes.AdditionalProperties);
-        if (value === 'drop' && exists) {
-          dropEvent = true;
-          violationType = violationTypes.AdditionalProperties;
-          break;
-        }
-        if (!(value === 'forward' || value === 'drop')) {
-          logger.error(`Unknown option ${value} in ${key}"`);
-        }
-        break;
-      }
-      case 'anyOtherViolation': {
-        const exists1 = violationsByType.has(violationTypes.UnknownViolation);
-        const exists2 = violationsByType.has(violationTypes.DatatypeMismatch);
-        const exists3 = violationsByType.has(violationTypes.RequiredMissing);
-        if (value === 'drop' && (exists1 || exists2 || exists3)) {
-          if (exists1) {
-            violationType = violationTypes.UnknownViolation;
-          } else if (exists2) {
-            violationType = violationTypes.DatatypeMismatch;
-          } else {
-            violationType = violationTypes.RequiredMissing;
-          }
-          dropEvent = true;
-          break;
-        }
-        if (!(value === 'forward' || value === 'drop')) {
-          logger.error(`Unknown option ${value} in ${key}"`);
-        }
-        break;
-      }
-      case 'sendViolatedEventsTo': {
-        if (value !== 'procerrors') {
-          logger.error(`Unknown option ${value} in ${key}"`);
-        }
-        break;
-      }
-    }
+    const value = mergedTpConfig[foundConfig]?.toString()?.toLowerCase();
+    const handler = handlerMap[foundConfig];
+    handler(value);
   }
+
   return { dropEvent, violationType };
 }
 
@@ -351,7 +365,7 @@ async function handleValidation(event) {
 
     ({ dropEvent, violationType } = handleValidationErrors(
       validationErrors,
-      mergedTpConfig,
+      event.metadata,
       dropEvent,
       violationType,
     ));