From d7861895d2e690900e83a7238f62b10d9e39ff91 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Fri, 1 Dec 2023 18:45:47 +0530 Subject: [PATCH 01/15] fix: removed retry logic from v1 cm360 proxy handler --- .../campaign_manager/networkHandler.js | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/src/v1/destinations/campaign_manager/networkHandler.js b/src/v1/destinations/campaign_manager/networkHandler.js index b3b82c8cf3..34ed87899a 100644 --- a/src/v1/destinations/campaign_manager/networkHandler.js +++ b/src/v1/destinations/campaign_manager/networkHandler.js @@ -10,25 +10,6 @@ const { } = require('../../../adapters/utils/networkUtils'); const tags = require('../../../v0/util/tags'); -function isEventRetryableAndExtractErrMsg(element, proxyOutputObj) { - let isRetryable = false; - let errorMsg = ''; - // success event - if (!element.errors) { - return isRetryable; - } - for (const err of element.errors) { - errorMsg += `${err.message}, `; - if (err.code === 'INTERNAL') { - isRetryable = true; - } - } - if (errorMsg) { - proxyOutputObj.error = errorMsg; - } - return isRetryable; -} - function isEventAbortableAndExtractErrMsg(element, proxyOutputObj) { let isAbortable = false; let errorMsg = ''; @@ -68,10 +49,8 @@ const responseHandler = (destinationResponse) => { metadata: rudderJobMetadata[idx], error: 'success', }; - // update status of partial event as per retriable or abortable - if (isEventRetryableAndExtractErrMsg(element, proxyOutputObj)) { - proxyOutputObj.statusCode = 500; - } else if (isEventAbortableAndExtractErrMsg(element, proxyOutputObj)) { + // update status of partial event if abortable + if (isEventAbortableAndExtractErrMsg(element, proxyOutputObj)) { proxyOutputObj.statusCode = 400; } responseWithIndividualEvents.push(proxyOutputObj); From a47de8b21c4719dcc3449c330d0ac49897deaa63 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Fri, 1 Dec 2023 21:13:45 +0530 Subject: [PATCH 02/15] fix: removed retry logic from v1 cm360 proxy handler --- src/adapters/networkHandlerFactory.js | 5 ++++- src/adapters/networkHandlerFactory.test.js | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/adapters/networkHandlerFactory.js b/src/adapters/networkHandlerFactory.js index f4940553f5..f1d9d14f57 100644 --- a/src/adapters/networkHandlerFactory.js +++ b/src/adapters/networkHandlerFactory.js @@ -37,7 +37,10 @@ SUPPORTED_VERSIONS.forEach((version) => { }); const getNetworkHandler = (type, version) => { - const NetworkHandler = handlers[version][type] || handlers.generic; + let NetworkHandler = handlers[version][type] || handlers.generic; + if (version === "v1" && NetworkHandler === handlers.generic) { + NetworkHandler = handlers.v0[type]; + } return new NetworkHandler(); }; diff --git a/src/adapters/networkHandlerFactory.test.js b/src/adapters/networkHandlerFactory.test.js index c4713e66a8..cc03222470 100644 --- a/src/adapters/networkHandlerFactory.test.js +++ b/src/adapters/networkHandlerFactory.test.js @@ -20,6 +20,8 @@ describe(`Network Handler Tests`, () => { it('Should return genericHandler if v1 proxy and handler is not present for destination', () => { let proxyHandler = getNetworkHandler('braze', `v1`); - expect(proxyHandler).toEqual(new GenericNetworkHandler()); + const brazeProxy = require(`../v0/destinations/braze/networkHandler`).networkHandler; + expect(proxyHandler).toEqual(new brazeProxy()); + // expect(proxyHandler).toEqual(new GenericNetworkHandler()); }); }); From 828a82f2f90d79db000f695d36462f84e779c57b Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Fri, 1 Dec 2023 21:15:48 +0530 Subject: [PATCH 03/15] fix: removed retry logic from v1 cm360 proxy handler --- src/adapters/networkHandlerFactory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/networkHandlerFactory.js b/src/adapters/networkHandlerFactory.js index f1d9d14f57..c32e8539ef 100644 --- a/src/adapters/networkHandlerFactory.js +++ b/src/adapters/networkHandlerFactory.js @@ -39,7 +39,7 @@ SUPPORTED_VERSIONS.forEach((version) => { const getNetworkHandler = (type, version) => { let NetworkHandler = handlers[version][type] || handlers.generic; if (version === "v1" && NetworkHandler === handlers.generic) { - NetworkHandler = handlers.v0[type]; + NetworkHandler = handlers.v0[type] || handlers.generic; } return new NetworkHandler(); }; From 31d19c7148e4782d06352f8c0ab3b718ba58545e Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 4 Dec 2023 16:12:36 +0530 Subject: [PATCH 04/15] feat: adapter to convert v1 payload to v0 transformer Proxy --- src/adapters/networkHandlerFactory.js | 7 ++- src/adapters/networkHandlerFactory.test.js | 22 +++---- src/controllers/delivery.ts | 58 ++++++++++++++----- src/services/delivertTest/deliveryTest.ts | 5 +- src/services/destination/nativeIntegration.ts | 58 +++++++++++++++++-- src/types/index.ts | 29 +++++++++- 6 files changed, 145 insertions(+), 34 deletions(-) diff --git a/src/adapters/networkHandlerFactory.js b/src/adapters/networkHandlerFactory.js index c32e8539ef..e8c3748d15 100644 --- a/src/adapters/networkHandlerFactory.js +++ b/src/adapters/networkHandlerFactory.js @@ -37,11 +37,14 @@ SUPPORTED_VERSIONS.forEach((version) => { }); const getNetworkHandler = (type, version) => { + let handlerVersion = version; let NetworkHandler = handlers[version][type] || handlers.generic; - if (version === "v1" && NetworkHandler === handlers.generic) { + if (version === 'v1' && NetworkHandler === handlers.generic) { NetworkHandler = handlers.v0[type] || handlers.generic; + handlerVersion = 'v0'; } - return new NetworkHandler(); + const networkHandler = new NetworkHandler(); + return { networkHandler, handlerVersion }; }; module.exports = { diff --git a/src/adapters/networkHandlerFactory.test.js b/src/adapters/networkHandlerFactory.test.js index cc03222470..8a3212b5bc 100644 --- a/src/adapters/networkHandlerFactory.test.js +++ b/src/adapters/networkHandlerFactory.test.js @@ -3,25 +3,27 @@ const { networkHandler: GenericNetworkHandler } = require('./networkhandler/gene describe(`Network Handler Tests`, () => { it('Should return v0 networkhandler', () => { - let proxyHandler = getNetworkHandler('campaign_manager', `v0`); + let { networkHandler, handlerVersion } = getNetworkHandler('campaign_manager', `v0`); const cmProxy = require(`../v0/destinations/campaign_manager/networkHandler`).networkHandler; - expect(proxyHandler).toEqual(new cmProxy()); + expect(networkHandler).toEqual(new cmProxy()); + }); - proxyHandler = getNetworkHandler('braze', `v0`); + it('Should return v0 networkhandler braze', () => { + let { networkHandler, handlerVersion } = getNetworkHandler('braze', `v0`); const brazeProxy = require(`../v0/destinations/braze/networkHandler`).networkHandler; - expect(proxyHandler).toEqual(new brazeProxy()); + expect(networkHandler).toEqual(new brazeProxy()); }); it('Should return v1 networkhandler', () => { - let proxyHandler = getNetworkHandler('campaign_manager', `v1`); + let { networkHandler, handlerVersion } = getNetworkHandler('campaign_manager', `v1`); const cmProxy = require(`../v1/destinations/campaign_manager/networkHandler`).networkHandler; - expect(proxyHandler).toEqual(new cmProxy()); + expect(networkHandler).toEqual(new cmProxy()); }); - it('Should return genericHandler if v1 proxy and handler is not present for destination', () => { - let proxyHandler = getNetworkHandler('braze', `v1`); + it('Should return v0 handler if v1 version and handler is present for destination in v0', () => { + const { networkHandler, handlerVersion } = getNetworkHandler('braze', `v1`); const brazeProxy = require(`../v0/destinations/braze/networkHandler`).networkHandler; - expect(proxyHandler).toEqual(new brazeProxy()); - // expect(proxyHandler).toEqual(new GenericNetworkHandler()); + console.log(networkHandler); + expect(networkHandler).toEqual(new brazeProxy()); }); }); diff --git a/src/controllers/delivery.ts b/src/controllers/delivery.ts index 3ccc241b87..1c0e3a5692 100644 --- a/src/controllers/delivery.ts +++ b/src/controllers/delivery.ts @@ -1,6 +1,13 @@ +/* eslint-disable prefer-destructuring */ +/* eslint-disable sonarjs/no-duplicate-string */ import { Context } from 'koa'; import { MiscService } from '../services/misc'; -import { DeliveryResponse, ProcessorTransformationOutput } from '../types/index'; +import { + DeliveryResponse, + ProcessorTransformationOutput, + ProcessorTransformationOutputWithMetaData, + ProcessorTransformationOutputWithMetaDataArray, +} from '../types/index'; import { ServiceSelector } from '../helpers/serviceSelector'; import { DeliveryTestService } from '../services/delivertTest/deliveryTest'; import { ControllerUtility } from './util'; @@ -14,9 +21,16 @@ export class DeliveryController { logger.debug('Native(Delivery):: Request to transformer::', JSON.stringify(ctx.request.body)); let deliveryResponse: DeliveryResponse; const requestMetadata = MiscService.getRequestMetadata(ctx); - const event = ctx.request.body as ProcessorTransformationOutput; - const { destination }: { destination: string } = ctx.params; const { version }: { version: string } = ctx.params; + let event: + | ProcessorTransformationOutputWithMetaData + | ProcessorTransformationOutputWithMetaDataArray; + if (version === 'v0') { + event = ctx.request.body as ProcessorTransformationOutputWithMetaData; + } else { + event = ctx.request.body as ProcessorTransformationOutputWithMetaDataArray; + } + const { destination }: { destination: string } = ctx.params; const integrationService = ServiceSelector.getNativeDestinationService(); try { deliveryResponse = await integrationService.deliver( @@ -26,17 +40,31 @@ export class DeliveryController { version, ); } catch (error: any) { - const metaTO = integrationService.getTags( - destination, - event.metadata?.destinationId || 'Non-determininable', - event.metadata?.workspaceId || 'Non-determininable', - tags.FEATURES.DATA_DELIVERY, - ); - metaTO.metadata = event.metadata; - deliveryResponse = DestinationPostTransformationService.handleDeliveryFailureEvents( - error, - metaTO, - ); + if (!Array.isArray(event.metadata)) { + const metaTO = integrationService.getTags( + destination, + event.metadata?.destinationId || 'Non-determininable', + event.metadata?.workspaceId || 'Non-determininable', + tags.FEATURES.DATA_DELIVERY, + ); + metaTO.metadata = event.metadata; + deliveryResponse = DestinationPostTransformationService.handleDeliveryFailureEvents( + error, + metaTO, + ); + } else { + const metaTO = integrationService.getTags( + destination, + event.metadata[0]?.destinationId || 'Non-determininable', + event.metadata[0]?.workspaceId || 'Non-determininable', + tags.FEATURES.DATA_DELIVERY, + ); + metaTO.metadata = event.metadata[0]; + deliveryResponse = DestinationPostTransformationService.handleDeliveryFailureEvents( + error, + metaTO, + ); + } } ctx.body = { output: deliveryResponse }; ControllerUtility.deliveryPostProcess(ctx, deliveryResponse.status); @@ -50,6 +78,7 @@ export class DeliveryController { JSON.stringify(ctx.request.body), ); const { destination }: { destination: string } = ctx.params; + const { version }: { version: string } = ctx.params; const { deliveryPayload, destinationRequestPayload, @@ -61,6 +90,7 @@ export class DeliveryController { destination, destinationRequestPayload, deliveryPayload, + version, ); ctx.body = { output: response }; ControllerUtility.postProcess(ctx); diff --git a/src/services/delivertTest/deliveryTest.ts b/src/services/delivertTest/deliveryTest.ts index 0d960ade17..2f5db183e5 100644 --- a/src/services/delivertTest/deliveryTest.ts +++ b/src/services/delivertTest/deliveryTest.ts @@ -12,12 +12,13 @@ export class DeliveryTestService { destination: string, routerDestReqPayload: any, routerDeliveryPayload: any, + version, ) { let response: any; try { - const destNetworkHandler = networkHandlerFactory.getNetworkHandler(destination); + const { networkHandler } = networkHandlerFactory.getNetworkHandler(destination, version); - const proxyDestReqPayload = destNetworkHandler.prepareProxy(routerDeliveryPayload); + const proxyDestReqPayload = networkHandler.prepareProxy(routerDeliveryPayload); response = { destinationRequestPayload: proxyDestReqPayload, }; diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 510fa80362..615562ecef 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -1,3 +1,5 @@ +/* eslint-disable prefer-destructuring */ +/* eslint-disable sonarjs/no-duplicate-string */ /* eslint-disable @typescript-eslint/no-unused-vars */ import groupBy from 'lodash/groupBy'; import cloneDeep from 'lodash/cloneDeep'; @@ -13,11 +15,14 @@ import { ProcessorTransformationOutput, UserDeletionRequest, UserDeletionResponse, + ProcessorTransformationOutputWithMetaData, + ProcessorTransformationOutputWithMetaDataArray, } from '../../types/index'; import { DestinationPostTransformationService } from './postTransformation'; import networkHandlerFactory from '../../adapters/networkHandlerFactory'; import { FetchHandler } from '../../helpers/fetchHandlers'; import tags from '../../v0/util/tags'; +import { ControllerUtility } from '../../controllers/util'; export class NativeIntegrationDestinationService implements DestinationService { public init() {} @@ -169,15 +174,48 @@ export class NativeIntegrationDestinationService implements DestinationService { } public async deliver( - destinationRequest: ProcessorTransformationOutput, + destinationRequest: + | ProcessorTransformationOutputWithMetaData + | ProcessorTransformationOutputWithMetaDataArray, destinationType: string, _requestMetadata: NonNullable, version: string, ): Promise { try { - const networkHandler = networkHandlerFactory.getNetworkHandler(destinationType, version); + const { networkHandler, handlerVersion } = networkHandlerFactory.getNetworkHandler( + destinationType, + version, + ); const rawProxyResponse = await networkHandler.proxy(destinationRequest, destinationType); const processedProxyResponse = networkHandler.processAxiosResponse(rawProxyResponse); + if ( + handlerVersion === 'v0' && + version === 'v1' && + Array.isArray(destinationRequest.metadata) + ) { + const [metadataZero] = destinationRequest.metadata; + const handlerResponse = networkHandler.responseHandler( + { + ...processedProxyResponse, + rudderJobMetadata: metadataZero, + }, + destinationType, + ) as DeliveryResponse; + + const responseWithIndividualEvents: { statusCode: number; metadata: any; error: string }[] = + []; + // eslint-disable-next-line no-restricted-syntax + for (const metadata of destinationRequest.metadata) { + responseWithIndividualEvents.push({ + statusCode: handlerResponse.status, + metadata, + error: handlerResponse.message, + }); + } + handlerResponse.response = responseWithIndividualEvents; + return handlerResponse; + } + return networkHandler.responseHandler( { ...processedProxyResponse, @@ -186,13 +224,23 @@ export class NativeIntegrationDestinationService implements DestinationService { destinationType, ) as DeliveryResponse; } catch (err: any) { + if (!Array.isArray(destinationRequest.metadata)) { + const metaTO = this.getTags( + destinationType, + destinationRequest.metadata?.destinationId || 'Non-determininable', + destinationRequest.metadata?.workspaceId || 'Non-determininable', + tags.FEATURES.DATA_DELIVERY, + ); + metaTO.metadata = destinationRequest.metadata; + return DestinationPostTransformationService.handleDeliveryFailureEvents(err, metaTO); + } const metaTO = this.getTags( destinationType, - destinationRequest.metadata?.destinationId || 'Non-determininable', - destinationRequest.metadata?.workspaceId || 'Non-determininable', + destinationRequest.metadata[0].destinationId || 'Non-determininable', + destinationRequest.metadata[0].workspaceId || 'Non-determininable', tags.FEATURES.DATA_DELIVERY, ); - metaTO.metadata = destinationRequest.metadata; + metaTO.metadata = destinationRequest.metadata[0]; return DestinationPostTransformationService.handleDeliveryFailureEvents(err, metaTO); } } diff --git a/src/types/index.ts b/src/types/index.ts index 7a23132173..d497811b6f 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -16,9 +16,34 @@ type ProcessorTransformationOutput = { FORM?: Record; }; files?: Record; - metadata?: Metadata; }; +// type ProcessorTransformationOutput = { +// version: string; +// type: string; +// method: string; +// endpoint: string; +// userId: string; +// headers?: Record; +// params?: Record; +// body?: { +// JSON?: Record; +// JSON_ARRAY?: Record; +// XML?: Record; +// FORM?: Record; +// }; +// files?: Record; +// metadata?: Metadata; +// }; + +type ProcessorTransformationOutputWithMetaData = { + metadata?: Metadata; +} & ProcessorTransformationOutput; + +type ProcessorTransformationOutputWithMetaDataArray = { + metadata?: Metadata[]; +} & ProcessorTransformationOutput; + type Metadata = { sourceId: string; workspaceId: string; @@ -276,4 +301,6 @@ export { ComparatorInput, SourceInput, Source, + ProcessorTransformationOutputWithMetaData, + ProcessorTransformationOutputWithMetaDataArray, }; From 222d46ab3fadecdb229c7c94ad80596452d8da15 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 4 Dec 2023 18:05:50 +0530 Subject: [PATCH 05/15] fix: changed type to any in handleDeliveryFailureEvents --- src/controllers/util/index.ts | 32 +++++++++++++++++++ src/services/destination/nativeIntegration.ts | 28 +++++----------- .../destination/postTransformation.ts | 18 +++-------- src/types/index.ts | 20 +----------- 4 files changed, 45 insertions(+), 53 deletions(-) diff --git a/src/controllers/util/index.ts b/src/controllers/util/index.ts index 75d3d8ffa7..7d857431c3 100644 --- a/src/controllers/util/index.ts +++ b/src/controllers/util/index.ts @@ -6,6 +6,8 @@ import get from 'get-value'; import { API_VERSION, CHANNELS, RETL_TIMESTAMP } from '../../routes/utils/constants'; import { getCompatibleStatusCode } from '../../adapters/utils/networkUtils'; import { + DeliveryResponse, + ProcessorTransformationOutputWithMetaDataArray, ProcessorTransformationRequest, RouterTransformationRequestData, RudderMessage, @@ -101,4 +103,34 @@ export class ControllerUtility { return { ...event, message: newMsg }; }); } + + public static convertV1ProxyPayloadToV0( + destinationRequest: ProcessorTransformationOutputWithMetaDataArray, + processedProxyResponse: any, + handlerVersion: any, + version: string, + networkHandler: any, + destinationType: string, + ) { + const [metadataZero] = destinationRequest.metadata; + const handlerResponse = networkHandler.responseHandler( + { + ...processedProxyResponse, + rudderJobMetadata: metadataZero, + }, + destinationType, + ) as DeliveryResponse; + + const responseWithIndividualEvents: { statusCode: number; metadata: any; error: string }[] = []; + // eslint-disable-next-line no-restricted-syntax + for (const metadata of destinationRequest.metadata) { + responseWithIndividualEvents.push({ + statusCode: handlerResponse.status, + metadata, + error: handlerResponse.message, + }); + } + handlerResponse.response = responseWithIndividualEvents; + return handlerResponse; + } } diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 615562ecef..60a96b3262 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -188,32 +188,20 @@ export class NativeIntegrationDestinationService implements DestinationService { ); const rawProxyResponse = await networkHandler.proxy(destinationRequest, destinationType); const processedProxyResponse = networkHandler.processAxiosResponse(rawProxyResponse); + if ( handlerVersion === 'v0' && version === 'v1' && Array.isArray(destinationRequest.metadata) ) { - const [metadataZero] = destinationRequest.metadata; - const handlerResponse = networkHandler.responseHandler( - { - ...processedProxyResponse, - rudderJobMetadata: metadataZero, - }, + return ControllerUtility.convertV1ProxyPayloadToV0( + destinationRequest as ProcessorTransformationOutputWithMetaDataArray, + processedProxyResponse, + handlerVersion, + version, + networkHandler, destinationType, - ) as DeliveryResponse; - - const responseWithIndividualEvents: { statusCode: number; metadata: any; error: string }[] = - []; - // eslint-disable-next-line no-restricted-syntax - for (const metadata of destinationRequest.metadata) { - responseWithIndividualEvents.push({ - statusCode: handlerResponse.status, - metadata, - error: handlerResponse.message, - }); - } - handlerResponse.response = responseWithIndividualEvents; - return handlerResponse; + ); } return networkHandler.responseHandler( diff --git a/src/services/destination/postTransformation.ts b/src/services/destination/postTransformation.ts index 1e99961045..cc7f1e387a 100644 --- a/src/services/destination/postTransformation.ts +++ b/src/services/destination/postTransformation.ts @@ -16,15 +16,6 @@ import { ErrorReportingService } from '../errorReporting'; import tags from '../../v0/util/tags'; import stats from '../../util/stats'; -type ErrorResponse = { - status?: number; - message?: string; - destinationResponse?: object; - statTags?: object; - authErrorCategory?: string | undefined; - response?: object | undefined; -}; - export class DestinationPostTransformationService { public static handleProcessorTransformSucessEvents( event: ProcessorTransformationRequest, @@ -148,7 +139,7 @@ export class DestinationPostTransformationService { } public static handleDeliveryFailureEvents( - error: ErrorResponse, + error: any, metaTo: MetaTransferObject, ): DeliveryResponse { const errObj = generateErrorObject(error, metaTo.errorDetails, false); @@ -162,10 +153,9 @@ export class DestinationPostTransformationService { }), } as DeliveryResponse; - // for transformer-proxy to maintain contract - const { response } = error; - if (response) { - resp.response = response; + // for transformer-proxy + if (error.response) { + resp.response = error.response; } ErrorReportingService.reportError(error, metaTo.errorContext, resp); return resp; diff --git a/src/types/index.ts b/src/types/index.ts index d497811b6f..0b33768476 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -18,30 +18,12 @@ type ProcessorTransformationOutput = { files?: Record; }; -// type ProcessorTransformationOutput = { -// version: string; -// type: string; -// method: string; -// endpoint: string; -// userId: string; -// headers?: Record; -// params?: Record; -// body?: { -// JSON?: Record; -// JSON_ARRAY?: Record; -// XML?: Record; -// FORM?: Record; -// }; -// files?: Record; -// metadata?: Metadata; -// }; - type ProcessorTransformationOutputWithMetaData = { metadata?: Metadata; } & ProcessorTransformationOutput; type ProcessorTransformationOutputWithMetaDataArray = { - metadata?: Metadata[]; + metadata: Metadata[]; } & ProcessorTransformationOutput; type Metadata = { From 70d098dbe170e3df82e9ee5255cc18897c51869c Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 4 Dec 2023 18:20:30 +0530 Subject: [PATCH 06/15] fix: changed type to any in handleDeliveryFailureEvents --- src/features.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/features.json b/src/features.json index 0d2d71bbd9..2d064eeae0 100644 --- a/src/features.json +++ b/src/features.json @@ -65,5 +65,5 @@ "TIKTOK_AUDIENCE": true }, "supportSourceTransformV1": true, - "supportTransformerProxyV1": false + "supportTransformerProxyV1": true } From db476b8a28aeccc7ccfeb6f66a87399046c09879 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 4 Dec 2023 18:58:23 +0530 Subject: [PATCH 07/15] fix: changed type to any in handleDeliveryFailureEvents --- src/v0/util/errorTypes/index.js | 2 ++ .../util/errorTypes/transformerProxyError.js | 28 +++++++++++++++++++ .../campaign_manager/networkHandler.js | 22 +++++++-------- 3 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 src/v0/util/errorTypes/transformerProxyError.js diff --git a/src/v0/util/errorTypes/index.js b/src/v0/util/errorTypes/index.js index 80268c3e77..f3cef657ca 100644 --- a/src/v0/util/errorTypes/index.js +++ b/src/v0/util/errorTypes/index.js @@ -1,5 +1,7 @@ const FilteredEventsError = require('./filteredEventsError'); +const TransformerProxyError = require('./transformerProxyError'); module.exports = { FilteredEventsError, + TransformerProxyError, }; diff --git a/src/v0/util/errorTypes/transformerProxyError.js b/src/v0/util/errorTypes/transformerProxyError.js new file mode 100644 index 0000000000..9f4bb593c7 --- /dev/null +++ b/src/v0/util/errorTypes/transformerProxyError.js @@ -0,0 +1,28 @@ +const tags = require('../tags'); +const { BaseError } = require('./base'); + +const errorTypes = Object.values(tags.ERROR_TYPES); +const metaTypes = Object.values(tags.METADATA); +class TransformerProxyError extends BaseError { + constructor(message, statusCode, statTags, destResponse, authErrorCategory, response) { + const finalStatTags = { + [tags.TAG_NAMES.ERROR_CATEGORY]: tags.ERROR_CATEGORIES.NETWORK, + [tags.TAG_NAMES.ERROR_TYPE]: tags.ERROR_TYPES.ABORTED, + }; + + // Allow specifying only error type and meta tags + if (statTags && typeof statTags === 'object' && !Array.isArray(statTags)) { + if (errorTypes.includes(statTags[tags.TAG_NAMES.ERROR_TYPE])) { + finalStatTags[tags.TAG_NAMES.ERROR_TYPE] = statTags[tags.TAG_NAMES.ERROR_TYPE]; + } + + if (metaTypes.includes(statTags[tags.TAG_NAMES.META])) { + finalStatTags[tags.TAG_NAMES.META] = statTags[tags.TAG_NAMES.META]; + } + } + super(message, statusCode, finalStatTags, destResponse, authErrorCategory); + this.response = response; + } +} + +module.exports = TransformerProxyError; \ No newline at end of file diff --git a/src/v1/destinations/campaign_manager/networkHandler.js b/src/v1/destinations/campaign_manager/networkHandler.js index 34ed87899a..6e13a68f0a 100644 --- a/src/v1/destinations/campaign_manager/networkHandler.js +++ b/src/v1/destinations/campaign_manager/networkHandler.js @@ -1,6 +1,6 @@ /* eslint-disable no-param-reassign */ /* eslint-disable no-restricted-syntax */ -const { NetworkError } = require('@rudderstack/integrations-lib'); +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); const { isHttpStatusSuccess, getAuthErrCategoryFromStCode } = require('../../../v0/util/index'); @@ -74,16 +74,16 @@ const responseHandler = (destinationResponse) => { }); } - throw new NetworkError( - `Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation`, - 500, - { - [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), - }, - destinationResponse, - getAuthErrCategoryFromStCode(status), - responseWithIndividualEvents, - ); + throw new TransformerProxyError( + `Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation`, + 500, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + getAuthErrCategoryFromStCode(status), + responseWithIndividualEvents, + ); }; function networkHandler() { From e790587a7b610d1f663d097a1789b09124d0f0c2 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 4 Dec 2023 23:36:34 +0530 Subject: [PATCH 08/15] fix: added response in error throw flow --- src/services/destination/nativeIntegration.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 60a96b3262..50e8d3a3f5 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -229,6 +229,21 @@ export class NativeIntegrationDestinationService implements DestinationService { tags.FEATURES.DATA_DELIVERY, ); metaTO.metadata = destinationRequest.metadata[0]; + + // if error is thrown and response is not in error, build response as per transformer proxy v1 + if (version === 'v1' && !err.response) { + const responseWithIndividualEvents: { statusCode: number; metadata: any; error: string }[] = + []; + // eslint-disable-next-line no-restricted-syntax + for (const metadata of destinationRequest.metadata) { + responseWithIndividualEvents.push({ + statusCode: err.status, + metadata, + error: err.message, + }); + } + err.response = responseWithIndividualEvents; + } return DestinationPostTransformationService.handleDeliveryFailureEvents(err, metaTO); } } From 878968a6eec8883c819c81c95cc764633cf84fab Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Tue, 5 Dec 2023 00:58:52 +0530 Subject: [PATCH 09/15] fix: comments addressed --- src/controllers/util/index.ts | 3 ++- src/services/destination/nativeIntegration.ts | 2 +- src/v0/destinations/campaign_manager/networkHandler.js | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/controllers/util/index.ts b/src/controllers/util/index.ts index 7d857431c3..a7f60712f7 100644 --- a/src/controllers/util/index.ts +++ b/src/controllers/util/index.ts @@ -127,7 +127,8 @@ export class ControllerUtility { responseWithIndividualEvents.push({ statusCode: handlerResponse.status, metadata, - error: handlerResponse.message, + error: + JSON.stringify(handlerResponse.destinationResponse.response) || handlerResponse.message, }); } handlerResponse.response = responseWithIndividualEvents; diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 50e8d3a3f5..338bf43451 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -239,7 +239,7 @@ export class NativeIntegrationDestinationService implements DestinationService { responseWithIndividualEvents.push({ statusCode: err.status, metadata, - error: err.message, + error: JSON.stringify(err.destinationResponse.response) || err.message, }); } err.response = responseWithIndividualEvents; diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index 63efff5b50..a1fa24835c 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -76,7 +76,7 @@ const responseHandler = (destinationResponse) => { throw new NetworkError( `Campaign Manager: ${response.error?.message} during CAMPAIGN_MANAGER response transformation 3`, - status, + 500, { [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), }, From 01fbeb1d23c741393206b4040d9e9eef8d763973 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Tue, 5 Dec 2023 01:12:48 +0530 Subject: [PATCH 10/15] fix: added component test for braze with v1 proxy --- src/adapters/networkHandlerFactory.test.js | 7 + .../destinations/braze/dataDelivery/data.ts | 925 ++++++++++-------- 2 files changed, 522 insertions(+), 410 deletions(-) diff --git a/src/adapters/networkHandlerFactory.test.js b/src/adapters/networkHandlerFactory.test.js index 8a3212b5bc..ff5f26a02d 100644 --- a/src/adapters/networkHandlerFactory.test.js +++ b/src/adapters/networkHandlerFactory.test.js @@ -26,4 +26,11 @@ describe(`Network Handler Tests`, () => { console.log(networkHandler); expect(networkHandler).toEqual(new brazeProxy()); }); + + it('Should return generic handler', () => { + const { networkHandler, handlerVersion } = getNetworkHandler('abc', `v1`); + const brazeProxy = require(`../v0/destinations/braze/networkHandler`).networkHandler; + console.log(networkHandler); + expect(networkHandler).toEqual(new GenericNetworkHandler()); + }); }); diff --git a/test/integrations/destinations/braze/dataDelivery/data.ts b/test/integrations/destinations/braze/dataDelivery/data.ts index 3e1805c01d..8e47fba0a1 100644 --- a/test/integrations/destinations/braze/dataDelivery/data.ts +++ b/test/integrations/destinations/braze/dataDelivery/data.ts @@ -1,395 +1,391 @@ -import MockAdapter from "axios-mock-adapter"; +import MockAdapter from 'axios-mock-adapter'; export const data = [ { - "name": "braze", - "description": "Test 0", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test1", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 0', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test1', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 200, - "body": { - "output": { - "status": 201, - "message": "Request for braze Processed Successfully", - "destinationResponse": { - "response": { - "aliases_processed": 1, - "message": "success" + output: { + response: { + status: 200, + body: { + output: { + status: 201, + message: 'Request for braze Processed Successfully', + destinationResponse: { + response: { + aliases_processed: 1, + message: 'success', }, - "status": 201 - } - } - } - } - } + status: 201, + }, + }, + }, + }, + }, }, { - "name": "braze", - "description": "Test 1", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test2", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 1', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test2', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 200, - "body": { - "output": { - "status": 201, - "message": "Request for braze Processed Successfully", - "destinationResponse": { - "response": { - "message": "success", - "errors": [ - "minor error message" - ] + output: { + response: { + status: 200, + body: { + output: { + status: 201, + message: 'Request for braze Processed Successfully', + destinationResponse: { + response: { + message: 'success', + errors: ['minor error message'], }, - "status": 201 - } - } - } - } - } + status: 201, + }, + }, + }, + }, + }, }, { - "name": "braze", - "description": "Test 2", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test3", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 2', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test3', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 200, - "body": { - "output": { - "status": 201, - "message": "Request failed for braze with status: 201", - "destinationResponse": { - "response": { - "message": "fatal error message", - "errors": [ - "minor error message" - ] + output: { + response: { + status: 200, + body: { + output: { + status: 201, + message: 'Request failed for braze with status: 201', + destinationResponse: { + response: { + message: 'fatal error message', + errors: ['minor error message'], }, - "status": 201 + status: 201, + }, + statTags: { + destType: 'BRAZE', + errorCategory: 'network', + destinationId: 'Non-determininable', + workspaceId: 'Non-determininable', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', }, - "statTags": { - "destType": "BRAZE", - "errorCategory": "network", - "destinationId": "Non-determininable", - "workspaceId": "Non-determininable", - "errorType": "aborted", - "feature": "dataDelivery", - "implementation": "native", - "module": "destination" - } - } - } - } - } + }, + }, + }, + }, }, { - "name": "braze", - "description": "Test 3", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test4", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 3', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test4', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', + }, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 201, + message: 'Request for braze Processed Successfully', + destinationResponse: { + response: '', + status: 201, }, - "JSON_ARRAY": {}, - "XML": {} }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + }, }, - "output": { - "response": { - "status": 200, - "body": { - "output": { - "status": 201, - "message": "Request for braze Processed Successfully", - "destinationResponse": { - "response": "", - "status": 201 - } - } - } - } - } }, { - "name": "braze", - "description": "Test 4", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test5", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 4', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test5', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 500, - "body": { - "output": { - "status": 500, - "message": "Request failed for braze with status: 500", - "destinationResponse": { - "response": "", - "status": 500 + output: { + response: { + status: 500, + body: { + output: { + status: 500, + message: 'Request failed for braze with status: 500', + destinationResponse: { + response: '', + status: 500, + }, + statTags: { + destType: 'BRAZE', + errorCategory: 'network', + destinationId: 'Non-determininable', + workspaceId: 'Non-determininable', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', }, - "statTags": { - "destType": "BRAZE", - "errorCategory": "network", - "destinationId": "Non-determininable", - "workspaceId": "Non-determininable", - "errorType": "retryable", - "feature": "dataDelivery", - "implementation": "native", - "module": "destination" - } - } - } - } - } + }, + }, + }, + }, }, { - "name": "braze", - "description": "Test 5", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test6", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 5', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test6', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 500, - "body": { - "output": { - "status": 500, - "message": "Request failed for braze with status: 500", - "destinationResponse": { - "response": "", - "status": 500 + output: { + response: { + status: 500, + body: { + output: { + status: 500, + message: 'Request failed for braze with status: 500', + destinationResponse: { + response: '', + status: 500, }, - "statTags": { - "destType": "BRAZE", - "errorCategory": "network", - "destinationId": "Non-determininable", - "workspaceId": "Non-determininable", - "errorType": "retryable", - "feature": "dataDelivery", - "implementation": "native", - "module": "destination" - } - } - } - } + statTags: { + destType: 'BRAZE', + errorCategory: 'network', + destinationId: 'Non-determininable', + workspaceId: 'Non-determininable', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + }, + }, + }, + }, }, mockFns: (mockAdapter: MockAdapter) => { // params has `{ destination: salesforce }` @@ -410,79 +406,188 @@ export const data = [ 'Content-Type': 'application/json', 'User-Agent': 'RudderLabs', }, - ).replyOnce((config) => { - // @ts-ignore - const err = AxiosError.from('DNS not found', 'ENOTFOUND', config) - return Promise.reject(err); + ) + .replyOnce((config) => { + // @ts-ignore + const err = AxiosError.from('DNS not found', 'ENOTFOUND', config); + return Promise.reject(err); }); - } + }, }, { - "name": "braze", - "description": "Test 6", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test7", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 6', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test7', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 500, - "body": { - "output": { - "status": 500, - "message": "Request failed for braze with status: 500", - "destinationResponse": { - "response": "", - "status": 500 + output: { + response: { + status: 500, + body: { + output: { + status: 500, + message: 'Request failed for braze with status: 500', + destinationResponse: { + response: '', + status: 500, + }, + statTags: { + destType: 'BRAZE', + errorCategory: 'network', + destinationId: 'Non-determininable', + workspaceId: 'Non-determininable', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', }, - "statTags": { - "destType": "BRAZE", - "errorCategory": "network", - "destinationId": "Non-determininable", - "workspaceId": "Non-determininable", - "errorType": "retryable", - "feature": "dataDelivery", - "implementation": "native", - "module": "destination" - } - } - } - } - } - } -] \ No newline at end of file + }, + }, + }, + }, + }, + { + name: 'braze', + description: 'Test Transformer Proxy V1 input with v0 proxy handler', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test1', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ + { + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + metadata: [ + { + jobId: 2, + attemptNum: 0, + userId: '', + sourceId: '2Vsge2uWYdrLfG7pZb5Y82eo4lr', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', + secret: { + access_token: 'secret', + refresh_token: 'refresh', + developer_token: 'developer_Token', + }, + }, + ], + files: {}, + params: { + destination: 'braze', + }, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 201, + message: 'Request for braze Processed Successfully', + destinationResponse: { + response: { + aliases_processed: 1, + message: 'success', + }, + status: 201, + rudderJobMetadata: { + jobId: 2, + attemptNum: 0, + userId: '', + sourceId: '2Vsge2uWYdrLfG7pZb5Y82eo4lr', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', + secret: { + access_token: 'secret', + refresh_token: 'refresh', + developer_token: 'developer_Token', + }, + }, + }, + response: [ + { + error: '{"aliases_processed":1,"message":"success"}', + statusCode: 201, + metadata: { + jobId: 2, + attemptNum: 0, + userId: '', + sourceId: '2Vsge2uWYdrLfG7pZb5Y82eo4lr', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', + secret: { + access_token: 'secret', + refresh_token: 'refresh', + developer_token: 'developer_Token', + }, + }, + }, + ], + }, + }, + }, + }, + }, +]; From 48e314b16672eaf6e8fb2024535bacb288bac40f Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Tue, 5 Dec 2023 14:09:45 +0530 Subject: [PATCH 11/15] feat: adding support fot v1 proxy contract --- jest.config.typescript.js | 1 + src/adapters/networkHandlerFactory.js | 10 ++- src/adapters/networkHandlerFactory.test.js | 29 ++++--- src/controllers/delivery.ts | 34 ++++++-- src/interfaces/DestinationService.ts | 4 +- src/services/comparator.ts | 4 +- src/services/destination/cdkV1Integration.ts | 4 +- src/services/destination/cdkV2Integration.ts | 12 ++- src/services/destination/nativeIntegration.ts | 32 ++++++-- .../destination/postTransformation.ts | 44 +++++++---- src/types/index.ts | 79 ++++++++++++++++--- 11 files changed, 191 insertions(+), 62 deletions(-) diff --git a/jest.config.typescript.js b/jest.config.typescript.js index fde50e3d5e..f058ac93af 100644 --- a/jest.config.typescript.js +++ b/jest.config.typescript.js @@ -184,4 +184,5 @@ module.exports = { // Whether to use watchman for file crawling // watchman: true, + setTimeout: 500000, }; diff --git a/src/adapters/networkHandlerFactory.js b/src/adapters/networkHandlerFactory.js index f4940553f5..e8c3748d15 100644 --- a/src/adapters/networkHandlerFactory.js +++ b/src/adapters/networkHandlerFactory.js @@ -37,8 +37,14 @@ SUPPORTED_VERSIONS.forEach((version) => { }); const getNetworkHandler = (type, version) => { - const NetworkHandler = handlers[version][type] || handlers.generic; - return new NetworkHandler(); + let handlerVersion = version; + let NetworkHandler = handlers[version][type] || handlers.generic; + if (version === 'v1' && NetworkHandler === handlers.generic) { + NetworkHandler = handlers.v0[type] || handlers.generic; + handlerVersion = 'v0'; + } + const networkHandler = new NetworkHandler(); + return { networkHandler, handlerVersion }; }; module.exports = { diff --git a/src/adapters/networkHandlerFactory.test.js b/src/adapters/networkHandlerFactory.test.js index c4713e66a8..ff5f26a02d 100644 --- a/src/adapters/networkHandlerFactory.test.js +++ b/src/adapters/networkHandlerFactory.test.js @@ -3,23 +3,34 @@ const { networkHandler: GenericNetworkHandler } = require('./networkhandler/gene describe(`Network Handler Tests`, () => { it('Should return v0 networkhandler', () => { - let proxyHandler = getNetworkHandler('campaign_manager', `v0`); + let { networkHandler, handlerVersion } = getNetworkHandler('campaign_manager', `v0`); const cmProxy = require(`../v0/destinations/campaign_manager/networkHandler`).networkHandler; - expect(proxyHandler).toEqual(new cmProxy()); + expect(networkHandler).toEqual(new cmProxy()); + }); - proxyHandler = getNetworkHandler('braze', `v0`); + it('Should return v0 networkhandler braze', () => { + let { networkHandler, handlerVersion } = getNetworkHandler('braze', `v0`); const brazeProxy = require(`../v0/destinations/braze/networkHandler`).networkHandler; - expect(proxyHandler).toEqual(new brazeProxy()); + expect(networkHandler).toEqual(new brazeProxy()); }); it('Should return v1 networkhandler', () => { - let proxyHandler = getNetworkHandler('campaign_manager', `v1`); + let { networkHandler, handlerVersion } = getNetworkHandler('campaign_manager', `v1`); const cmProxy = require(`../v1/destinations/campaign_manager/networkHandler`).networkHandler; - expect(proxyHandler).toEqual(new cmProxy()); + expect(networkHandler).toEqual(new cmProxy()); }); - it('Should return genericHandler if v1 proxy and handler is not present for destination', () => { - let proxyHandler = getNetworkHandler('braze', `v1`); - expect(proxyHandler).toEqual(new GenericNetworkHandler()); + it('Should return v0 handler if v1 version and handler is present for destination in v0', () => { + const { networkHandler, handlerVersion } = getNetworkHandler('braze', `v1`); + const brazeProxy = require(`../v0/destinations/braze/networkHandler`).networkHandler; + console.log(networkHandler); + expect(networkHandler).toEqual(new brazeProxy()); + }); + + it('Should return generic handler', () => { + const { networkHandler, handlerVersion } = getNetworkHandler('abc', `v1`); + const brazeProxy = require(`../v0/destinations/braze/networkHandler`).networkHandler; + console.log(networkHandler); + expect(networkHandler).toEqual(new GenericNetworkHandler()); }); }); diff --git a/src/controllers/delivery.ts b/src/controllers/delivery.ts index 3ccc241b87..0a6db25ac6 100644 --- a/src/controllers/delivery.ts +++ b/src/controllers/delivery.ts @@ -1,6 +1,13 @@ import { Context } from 'koa'; import { MiscService } from '../services/misc'; -import { DeliveryResponse, ProcessorTransformationOutput } from '../types/index'; +import { + DeliveriesResponse, + DeliveryResponse, + ProcessorTransformationOutput, + ProxyDeliveriesRequest, + ProxyDeliveryRequest, + ProxyRequest, +} from '../types/index'; import { ServiceSelector } from '../helpers/serviceSelector'; import { DeliveryTestService } from '../services/delivertTest/deliveryTest'; import { ControllerUtility } from './util'; @@ -12,34 +19,45 @@ import { FixMe } from '../util/types'; export class DeliveryController { public static async deliverToDestination(ctx: Context) { logger.debug('Native(Delivery):: Request to transformer::', JSON.stringify(ctx.request.body)); - let deliveryResponse: DeliveryResponse; + let deliveryResponse: DeliveryResponse | DeliveriesResponse; const requestMetadata = MiscService.getRequestMetadata(ctx); - const event = ctx.request.body as ProcessorTransformationOutput; + const deliveryRequest = ctx.request.body as ProxyRequest; const { destination }: { destination: string } = ctx.params; const { version }: { version: string } = ctx.params; const integrationService = ServiceSelector.getNativeDestinationService(); try { deliveryResponse = await integrationService.deliver( - event, + deliveryRequest, destination, requestMetadata, version, ); } catch (error: any) { + const metadata = Array.isArray(deliveryRequest.metadata) + ? deliveryRequest.metadata[0] + : deliveryRequest.metadata; const metaTO = integrationService.getTags( destination, - event.metadata?.destinationId || 'Non-determininable', - event.metadata?.workspaceId || 'Non-determininable', + metadata?.destinationId || 'Non-determinable', + metadata?.workspaceId || 'Non-determinable', tags.FEATURES.DATA_DELIVERY, ); - metaTO.metadata = event.metadata; + if (version.toLowerCase() === 'v1') { + metaTO.metadatas = (deliveryRequest as ProxyDeliveriesRequest).metadata; + } else { + metaTO.metadata = (deliveryRequest as ProxyDeliveryRequest).metadata; + } deliveryResponse = DestinationPostTransformationService.handleDeliveryFailureEvents( error, metaTO, ); } ctx.body = { output: deliveryResponse }; - ControllerUtility.deliveryPostProcess(ctx, deliveryResponse.status); + if (version.toLowerCase() === 'v1') { + ControllerUtility.deliveryPostProcess(ctx); + } else { + ControllerUtility.deliveryPostProcess(ctx, deliveryResponse.status); + } logger.debug('Native(Delivery):: Response from transformer::', JSON.stringify(ctx.body)); return ctx; } diff --git a/src/interfaces/DestinationService.ts b/src/interfaces/DestinationService.ts index 16f6b9349c..f30a9c42f1 100644 --- a/src/interfaces/DestinationService.ts +++ b/src/interfaces/DestinationService.ts @@ -5,9 +5,9 @@ import { ProcessorTransformationResponse, RouterTransformationRequestData, RouterTransformationResponse, - ProcessorTransformationOutput, UserDeletionRequest, UserDeletionResponse, + ProxyRequest, } from '../types/index'; export interface DestinationService { @@ -44,7 +44,7 @@ export interface DestinationService { ): RouterTransformationResponse[]; deliver( - event: ProcessorTransformationOutput, + event: ProxyRequest, destinationType: string, requestMetadata: NonNullable, version: string, diff --git a/src/services/comparator.ts b/src/services/comparator.ts index 58c96beabb..e551086b02 100644 --- a/src/services/comparator.ts +++ b/src/services/comparator.ts @@ -5,9 +5,9 @@ import { Destination, ErrorDetailer, MetaTransferObject, - ProcessorTransformationOutput, ProcessorTransformationRequest, ProcessorTransformationResponse, + ProxyRequest, RouterTransformationRequestData, RouterTransformationResponse, UserDeletionRequest, @@ -365,7 +365,7 @@ export class ComparatorService implements DestinationService { } public async deliver( - event: ProcessorTransformationOutput, + event: ProxyRequest, destinationType: string, requestMetadata: NonNullable, version: string, diff --git a/src/services/destination/cdkV1Integration.ts b/src/services/destination/cdkV1Integration.ts index 8ccd3341e5..1fab437d15 100644 --- a/src/services/destination/cdkV1Integration.ts +++ b/src/services/destination/cdkV1Integration.ts @@ -11,9 +11,9 @@ import { ProcessorTransformationResponse, RouterTransformationRequestData, RouterTransformationResponse, - ProcessorTransformationOutput, UserDeletionRequest, UserDeletionResponse, + ProxyRequest, } from '../../types/index'; import { DestinationPostTransformationService } from './postTransformation'; import tags from '../../v0/util/tags'; @@ -117,7 +117,7 @@ export class CDKV1DestinationService implements DestinationService { } public deliver( - _event: ProcessorTransformationOutput, + _event: ProxyRequest, _destinationType: string, _requestMetadata: NonNullable, ): Promise { diff --git a/src/services/destination/cdkV2Integration.ts b/src/services/destination/cdkV2Integration.ts index f3be2c0144..da6a15d54b 100644 --- a/src/services/destination/cdkV2Integration.ts +++ b/src/services/destination/cdkV2Integration.ts @@ -15,6 +15,7 @@ import { ProcessorTransformationOutput, UserDeletionRequest, UserDeletionResponse, + ProxyRequest, } from '../../types/index'; import tags from '../../v0/util/tags'; import { DestinationPostTransformationService } from './postTransformation'; @@ -64,7 +65,7 @@ export class CDKV2DestinationService implements DestinationService { destinationType, event, tags.FEATURES.PROCESSOR, - requestMetadata + requestMetadata, ); stats.increment('event_transform_success', { @@ -127,7 +128,12 @@ export class CDKV2DestinationService implements DestinationService { metaTo.metadata = destInputArray[0].metadata; try { const doRouterTransformationResponse: RouterTransformationResponse[] = - await processCdkV2Workflow(destinationType, destInputArray, tags.FEATURES.ROUTER, requestMetadata); + await processCdkV2Workflow( + destinationType, + destInputArray, + tags.FEATURES.ROUTER, + requestMetadata, + ); return DestinationPostTransformationService.handleRouterTransformSuccessEvents( doRouterTransformationResponse, undefined, @@ -160,7 +166,7 @@ export class CDKV2DestinationService implements DestinationService { } public deliver( - _event: ProcessorTransformationOutput, + _event: ProxyRequest, _destinationType: string, _requestMetadata: NonNullable, ): Promise { diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 510fa80362..2f70637245 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -13,6 +13,9 @@ import { ProcessorTransformationOutput, UserDeletionRequest, UserDeletionResponse, + ProxyRequest, + ProxyDeliveriesRequest, + ProxyDeliveryRequest, } from '../../types/index'; import { DestinationPostTransformationService } from './postTransformation'; import networkHandlerFactory from '../../adapters/networkHandlerFactory'; @@ -169,30 +172,45 @@ export class NativeIntegrationDestinationService implements DestinationService { } public async deliver( - destinationRequest: ProcessorTransformationOutput, + deliveryRequest: ProxyRequest, destinationType: string, _requestMetadata: NonNullable, version: string, ): Promise { try { - const networkHandler = networkHandlerFactory.getNetworkHandler(destinationType, version); - const rawProxyResponse = await networkHandler.proxy(destinationRequest, destinationType); + const { networkHandler, handlerVersion } = networkHandlerFactory.getNetworkHandler( + destinationType, + version, + ); + const rawProxyResponse = await networkHandler.proxy(deliveryRequest, destinationType); const processedProxyResponse = networkHandler.processAxiosResponse(rawProxyResponse); + const rudderJobMetadata = + handlerVersion.toLowerCase() === 'v1' + ? (deliveryRequest as ProxyDeliveriesRequest).metadata + : (deliveryRequest as ProxyDeliveryRequest).metadata; + return networkHandler.responseHandler( { ...processedProxyResponse, - rudderJobMetadata: destinationRequest.metadata, + rudderJobMetadata, }, destinationType, ) as DeliveryResponse; } catch (err: any) { + const metadata = Array.isArray(deliveryRequest.metadata) + ? deliveryRequest.metadata[0] + : deliveryRequest.metadata; const metaTO = this.getTags( destinationType, - destinationRequest.metadata?.destinationId || 'Non-determininable', - destinationRequest.metadata?.workspaceId || 'Non-determininable', + metadata?.destinationId || 'Non-determininable', + metadata?.workspaceId || 'Non-determininable', tags.FEATURES.DATA_DELIVERY, ); - metaTO.metadata = destinationRequest.metadata; + if (version.toLowerCase() === 'v1') { + metaTO.metadatas = (deliveryRequest as ProxyDeliveriesRequest).metadata; + } else { + metaTO.metadata = (deliveryRequest as ProxyDeliveryRequest).metadata; + } return DestinationPostTransformationService.handleDeliveryFailureEvents(err, metaTO); } } diff --git a/src/services/destination/postTransformation.ts b/src/services/destination/postTransformation.ts index 1e99961045..5597337eab 100644 --- a/src/services/destination/postTransformation.ts +++ b/src/services/destination/postTransformation.ts @@ -2,6 +2,7 @@ import cloneDeep from 'lodash/cloneDeep'; import isObject from 'lodash/isObject'; import isEmpty from 'lodash/isEmpty'; +import { PlatformError } from '@rudderstack/integrations-lib'; import { ProcessorTransformationRequest, ProcessorTransformationResponse, @@ -10,21 +11,14 @@ import { DeliveryResponse, MetaTransferObject, UserDeletionResponse, + DeliveriesResponse, + DeliveryJobState, } from '../../types/index'; import { generateErrorObject } from '../../v0/util'; import { ErrorReportingService } from '../errorReporting'; import tags from '../../v0/util/tags'; import stats from '../../util/stats'; -type ErrorResponse = { - status?: number; - message?: string; - destinationResponse?: object; - statTags?: object; - authErrorCategory?: string | undefined; - response?: object | undefined; -}; - export class DestinationPostTransformationService { public static handleProcessorTransformSucessEvents( event: ProcessorTransformationRequest, @@ -148,7 +142,7 @@ export class DestinationPostTransformationService { } public static handleDeliveryFailureEvents( - error: ErrorResponse, + error: NonNullable, metaTo: MetaTransferObject, ): DeliveryResponse { const errObj = generateErrorObject(error, metaTo.errorDetails, false); @@ -161,12 +155,34 @@ export class DestinationPostTransformationService { authErrorCategory: errObj.authErrorCategory, }), } as DeliveryResponse; + ErrorReportingService.reportError(error, metaTo.errorContext, resp); + return resp; + } - // for transformer-proxy to maintain contract - const { response } = error; - if (response) { - resp.response = response; + public static handlevV1DeliveriesFailureEvents( + error: NonNullable, + metaTo: MetaTransferObject, + ): DeliveriesResponse { + const errObj = generateErrorObject(error, metaTo.errorDetails, false); + const metadataArray = metaTo.metadatas; + if (!Array.isArray(metadataArray)) { + // Panic + throw new PlatformError('Proxy v1 endpoint error : metadataArray is not an array'); } + const responses = metadataArray.map((metadata) => { + const resp = { + error: errObj.message || '[Delivery] Error occured while processing payload', + statusCode: errObj.status, + metadata, + } as DeliveryJobState; + return resp; + }); + + const resp = { + responses, + statTags: errObj.statTags, + } as DeliveriesResponse; + ErrorReportingService.reportError(error, metaTo.errorContext, resp); return resp; } diff --git a/src/types/index.ts b/src/types/index.ts index 7a23132173..857ed6ec4c 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -19,6 +19,44 @@ type ProcessorTransformationOutput = { metadata?: Metadata; }; +type ProxyDeliveryRequest = { + version: string; + type: string; + method: string; + endpoint: string; + userId: string; + headers?: Record; + params?: Record; + body?: { + JSON?: Record; + JSON_ARRAY?: Record; + XML?: Record; + FORM?: Record; + }; + files?: Record; + metadata: Metadata; +}; + +type ProxyDeliveriesRequest = { + version: string; + type: string; + method: string; + endpoint: string; + userId: string; + headers?: Record; + params?: Record; + body?: { + JSON?: Record; + JSON_ARRAY?: Record; + XML?: Record; + FORM?: Record; + }; + files?: Record; + metadata: Metadata[]; +}; + +type ProxyRequest = ProxyDeliveryRequest | ProxyDeliveriesRequest; + type Metadata = { sourceId: string; workspaceId: string; @@ -50,6 +88,7 @@ type Metadata = { sourceDefinitionId: string; destinationDefinitionId: string; transformationId: string; + dontBatch?: boolean; }; type MessageIdMetadataMap = { @@ -140,7 +179,16 @@ type DeliveryResponse = { destinationResponse: any; statTags: object; authErrorCategory?: string; - response?: object; +}; + +type DeliveryJobState = { + error: string; + statusCode: number; + metadata: Metadata; +}; + +type DeliveriesResponse = { + responses: DeliveryJobState[]; }; enum MessageType { @@ -254,26 +302,31 @@ type SourceInput = { source?: Source; }; export { - Metadata, + ComparatorInput, + DeliveryJobState, + DeliveryResponse, + DeliveriesResponse, + Destination, + ErrorDetailer, MessageIdMetadataMap, - UserTransformationLibrary, + MetaTransferObject, + Metadata, + ProcessorTransformationOutput, ProcessorTransformationRequest, ProcessorTransformationResponse, + ProxyDeliveriesRequest, + ProxyDeliveryRequest, + ProxyRequest, RouterTransformationRequest, RouterTransformationRequestData, RouterTransformationResponse, RudderMessage, - ProcessorTransformationOutput, + Source, + SourceInput, SourceTransformationResponse, - DeliveryResponse, - ErrorDetailer, - UserTransformationResponse, - UserTransformationServiceResponse, - MetaTransferObject, UserDeletionRequest, UserDeletionResponse, - Destination, - ComparatorInput, - SourceInput, - Source, + UserTransformationLibrary, + UserTransformationResponse, + UserTransformationServiceResponse, }; From 6c544e8fd967e3f1399fc75cd066e99f57cf3d6a Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Tue, 5 Dec 2023 14:11:46 +0530 Subject: [PATCH 12/15] feat: cleanup --- jest.config.typescript.js | 1 - 1 file changed, 1 deletion(-) diff --git a/jest.config.typescript.js b/jest.config.typescript.js index f058ac93af..fde50e3d5e 100644 --- a/jest.config.typescript.js +++ b/jest.config.typescript.js @@ -184,5 +184,4 @@ module.exports = { // Whether to use watchman for file crawling // watchman: true, - setTimeout: 500000, }; From 29727dfa675a994f4af6480029d8b160708c9e9e Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Thu, 7 Dec 2023 20:03:21 +0530 Subject: [PATCH 13/15] chore: handle the v0->v1 adpation logic for delivery --- src/controllers/delivery.ts | 68 +++++++++++++------ src/interfaces/DestinationService.ts | 3 +- src/routes/delivery.ts | 9 ++- src/services/comparator.ts | 4 +- src/services/destination/cdkV1Integration.ts | 3 +- src/services/destination/cdkV2Integration.ts | 3 +- src/services/destination/nativeIntegration.ts | 28 ++++++-- 7 files changed, 86 insertions(+), 32 deletions(-) diff --git a/src/controllers/delivery.ts b/src/controllers/delivery.ts index 0a6db25ac6..67ec006058 100644 --- a/src/controllers/delivery.ts +++ b/src/controllers/delivery.ts @@ -6,7 +6,6 @@ import { ProcessorTransformationOutput, ProxyDeliveriesRequest, ProxyDeliveryRequest, - ProxyRequest, } from '../types/index'; import { ServiceSelector } from '../helpers/serviceSelector'; import { DeliveryTestService } from '../services/delivertTest/deliveryTest'; @@ -16,48 +15,75 @@ import { DestinationPostTransformationService } from '../services/destination/po import tags from '../v0/util/tags'; import { FixMe } from '../util/types'; +const NON_DETERMINABLE = 'Non-determinable'; + export class DeliveryController { public static async deliverToDestination(ctx: Context) { logger.debug('Native(Delivery):: Request to transformer::', JSON.stringify(ctx.request.body)); - let deliveryResponse: DeliveryResponse | DeliveriesResponse; + let deliveryResponse: DeliveryResponse; const requestMetadata = MiscService.getRequestMetadata(ctx); - const deliveryRequest = ctx.request.body as ProxyRequest; + const deliveryRequest = ctx.request.body as ProxyDeliveryRequest; const { destination }: { destination: string } = ctx.params; - const { version }: { version: string } = ctx.params; const integrationService = ServiceSelector.getNativeDestinationService(); try { - deliveryResponse = await integrationService.deliver( + deliveryResponse = (await integrationService.deliver( deliveryRequest, destination, requestMetadata, - version, - ); + 'v0', + )) as DeliveryResponse; } catch (error: any) { - const metadata = Array.isArray(deliveryRequest.metadata) - ? deliveryRequest.metadata[0] - : deliveryRequest.metadata; + const { metadata } = deliveryRequest; const metaTO = integrationService.getTags( destination, - metadata?.destinationId || 'Non-determinable', - metadata?.workspaceId || 'Non-determinable', + metadata?.destinationId || NON_DETERMINABLE, + metadata?.workspaceId || NON_DETERMINABLE, tags.FEATURES.DATA_DELIVERY, ); - if (version.toLowerCase() === 'v1') { - metaTO.metadatas = (deliveryRequest as ProxyDeliveriesRequest).metadata; - } else { - metaTO.metadata = (deliveryRequest as ProxyDeliveryRequest).metadata; - } + metaTO.metadata = metadata; deliveryResponse = DestinationPostTransformationService.handleDeliveryFailureEvents( error, metaTO, ); } ctx.body = { output: deliveryResponse }; - if (version.toLowerCase() === 'v1') { - ControllerUtility.deliveryPostProcess(ctx); - } else { - ControllerUtility.deliveryPostProcess(ctx, deliveryResponse.status); + ControllerUtility.deliveryPostProcess(ctx, deliveryResponse.status); + + logger.debug('Native(Delivery):: Response from transformer::', JSON.stringify(ctx.body)); + return ctx; + } + + public static async deliverToDestinationV1(ctx: Context) { + logger.debug('Native(Delivery):: Request to transformer::', JSON.stringify(ctx.request.body)); + let deliveryResponse: DeliveriesResponse; + const requestMetadata = MiscService.getRequestMetadata(ctx); + const deliveryRequest = ctx.request.body as ProxyDeliveriesRequest; + const { destination }: { destination: string } = ctx.params; + const integrationService = ServiceSelector.getNativeDestinationService(); + try { + deliveryResponse = (await integrationService.deliver( + deliveryRequest, + destination, + requestMetadata, + 'v1', + )) as DeliveriesResponse; + } catch (error: any) { + const { metadata } = deliveryRequest; + const metaTO = integrationService.getTags( + destination, + metadata[0].destinationId || NON_DETERMINABLE, + metadata[0].workspaceId || NON_DETERMINABLE, + tags.FEATURES.DATA_DELIVERY, + ); + metaTO.metadatas = metadata; + deliveryResponse = DestinationPostTransformationService.handlevV1DeliveriesFailureEvents( + error, + metaTO, + ); } + ctx.body = { output: deliveryResponse }; + ControllerUtility.deliveryPostProcess(ctx); + logger.debug('Native(Delivery):: Response from transformer::', JSON.stringify(ctx.body)); return ctx; } diff --git a/src/interfaces/DestinationService.ts b/src/interfaces/DestinationService.ts index f30a9c42f1..bf39024d85 100644 --- a/src/interfaces/DestinationService.ts +++ b/src/interfaces/DestinationService.ts @@ -8,6 +8,7 @@ import { UserDeletionRequest, UserDeletionResponse, ProxyRequest, + DeliveriesResponse, } from '../types/index'; export interface DestinationService { @@ -48,7 +49,7 @@ export interface DestinationService { destinationType: string, requestMetadata: NonNullable, version: string, - ): Promise; + ): Promise; processUserDeletion( requests: UserDeletionRequest[], diff --git a/src/routes/delivery.ts b/src/routes/delivery.ts index 0591dc8b9e..30f4e2fb78 100644 --- a/src/routes/delivery.ts +++ b/src/routes/delivery.ts @@ -5,12 +5,19 @@ import { RouteActivationMiddleware } from '../middlewares/routeActivation'; const router = new Router(); router.post( - '/:version/destinations/:destination/proxy', + '/v0/destinations/:destination/proxy', RouteActivationMiddleware.isDeliveryRouteActive, RouteActivationMiddleware.destinationDeliveryFilter, DeliveryController.deliverToDestination, ); +router.post( + '/v1/destinations/:destination/proxy', + RouteActivationMiddleware.isDeliveryRouteActive, + RouteActivationMiddleware.destinationDeliveryFilter, + DeliveryController.deliverToDestinationV1, +); + router.post( '/:version/destinations/:destination/proxyTest', RouteActivationMiddleware.isDeliveryTestRouteActive, diff --git a/src/services/comparator.ts b/src/services/comparator.ts index e551086b02..d1e085b4bd 100644 --- a/src/services/comparator.ts +++ b/src/services/comparator.ts @@ -1,6 +1,7 @@ /* eslint-disable class-methods-use-this */ import { DestinationService } from '../interfaces/DestinationService'; import { + DeliveriesResponse, DeliveryResponse, Destination, ErrorDetailer, @@ -369,7 +370,7 @@ export class ComparatorService implements DestinationService { destinationType: string, requestMetadata: NonNullable, version: string, - ): Promise { + ): Promise { const primaryResplist = await this.primaryService.deliver( event, destinationType, @@ -377,7 +378,6 @@ export class ComparatorService implements DestinationService { version, ); logger.error('[LIVE_COMPARE_TEST] not implemented for delivery routine'); - return primaryResplist; } diff --git a/src/services/destination/cdkV1Integration.ts b/src/services/destination/cdkV1Integration.ts index 1fab437d15..197e3162ea 100644 --- a/src/services/destination/cdkV1Integration.ts +++ b/src/services/destination/cdkV1Integration.ts @@ -14,6 +14,7 @@ import { UserDeletionRequest, UserDeletionResponse, ProxyRequest, + DeliveriesResponse, } from '../../types/index'; import { DestinationPostTransformationService } from './postTransformation'; import tags from '../../v0/util/tags'; @@ -120,7 +121,7 @@ export class CDKV1DestinationService implements DestinationService { _event: ProxyRequest, _destinationType: string, _requestMetadata: NonNullable, - ): Promise { + ): Promise { throw new TransformationError('CDV1 Does not Implement Delivery Routine'); } diff --git a/src/services/destination/cdkV2Integration.ts b/src/services/destination/cdkV2Integration.ts index da6a15d54b..be7f0e51d5 100644 --- a/src/services/destination/cdkV2Integration.ts +++ b/src/services/destination/cdkV2Integration.ts @@ -16,6 +16,7 @@ import { UserDeletionRequest, UserDeletionResponse, ProxyRequest, + DeliveriesResponse, } from '../../types/index'; import tags from '../../v0/util/tags'; import { DestinationPostTransformationService } from './postTransformation'; @@ -169,7 +170,7 @@ export class CDKV2DestinationService implements DestinationService { _event: ProxyRequest, _destinationType: string, _requestMetadata: NonNullable, - ): Promise { + ): Promise { throw new TransformationError('CDKV2 Does not Implement Delivery Routine'); } diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 2f70637245..a4f2969b7b 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -16,6 +16,8 @@ import { ProxyRequest, ProxyDeliveriesRequest, ProxyDeliveryRequest, + DeliveriesResponse, + DeliveryJobState, } from '../../types/index'; import { DestinationPostTransformationService } from './postTransformation'; import networkHandlerFactory from '../../adapters/networkHandlerFactory'; @@ -176,7 +178,7 @@ export class NativeIntegrationDestinationService implements DestinationService { destinationType: string, _requestMetadata: NonNullable, version: string, - ): Promise { + ): Promise { try { const { networkHandler, handlerVersion } = networkHandlerFactory.getNetworkHandler( destinationType, @@ -189,13 +191,29 @@ export class NativeIntegrationDestinationService implements DestinationService { ? (deliveryRequest as ProxyDeliveriesRequest).metadata : (deliveryRequest as ProxyDeliveryRequest).metadata; - return networkHandler.responseHandler( + let response = networkHandler.responseHandler( { ...processedProxyResponse, rudderJobMetadata, }, destinationType, - ) as DeliveryResponse; + ); + // Adaption Logic for V0 to V1 + if (handlerVersion.toLowerCase() === 'v0' && version.toLowerCase() === 'v1') { + const v0Response = response as DeliveryResponse; + const jobStates = (deliveryRequest as ProxyDeliveriesRequest).metadata.map( + (metadata) => + ({ + error: JSON.stringify(v0Response.destinationResponse), + statusCode: v0Response.status, + metadata, + } as DeliveryJobState), + ); + response = { + responses: jobStates, + } as DeliveriesResponse; + } + return response; } catch (err: any) { const metadata = Array.isArray(deliveryRequest.metadata) ? deliveryRequest.metadata[0] @@ -208,9 +226,9 @@ export class NativeIntegrationDestinationService implements DestinationService { ); if (version.toLowerCase() === 'v1') { metaTO.metadatas = (deliveryRequest as ProxyDeliveriesRequest).metadata; - } else { - metaTO.metadata = (deliveryRequest as ProxyDeliveryRequest).metadata; + return DestinationPostTransformationService.handlevV1DeliveriesFailureEvents(err, metaTO); } + metaTO.metadata = (deliveryRequest as ProxyDeliveryRequest).metadata; return DestinationPostTransformationService.handleDeliveryFailureEvents(err, metaTO); } } From 49ba17c02c521e48c3832c070519ab4b268fd284 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 11 Dec 2023 11:30:20 +0530 Subject: [PATCH 14/15] fix: removed response building from catch block --- src/services/destination/nativeIntegration.ts | 16 --- .../destinations/braze/dataDelivery/data.ts | 114 +++++++++++++++--- .../destinations/braze/network.ts | 29 +++++ 3 files changed, 123 insertions(+), 36 deletions(-) diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 0b175686e2..c3b0237cdc 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -217,7 +217,6 @@ export class NativeIntegrationDestinationService implements DestinationService { } as DeliveryJobState), ); responseProxy = { - ...responseProxy, response: jobStates, } as DeliveriesResponse; } @@ -233,21 +232,6 @@ export class NativeIntegrationDestinationService implements DestinationService { tags.FEATURES.DATA_DELIVERY, ); - // if error is thrown and response is not in error, build response as per transformer proxy v1 - if (version === 'v1' && !err.response && Array.isArray(metadata)) { - const responseWithIndividualEvents: { statusCode: number; metadata: any; error: string }[] = - []; - // eslint-disable-next-line no-restricted-syntax - for (const meta of metadata) { - responseWithIndividualEvents.push({ - statusCode: err.status, - metadata: meta, - error: JSON.stringify(err.destinationResponse.response) || err.message, - }); - } - err.response = responseWithIndividualEvents; - } - if (version.toLowerCase() === 'v1') { metaTO.metadatas = (deliveryRequest as ProxyDeliveriesRequest).metadata; return DestinationPostTransformationService.handlevV1DeliveriesFailureEvents(err, metaTO); diff --git a/test/integrations/destinations/braze/dataDelivery/data.ts b/test/integrations/destinations/braze/dataDelivery/data.ts index 8e47fba0a1..cd688a621b 100644 --- a/test/integrations/destinations/braze/dataDelivery/data.ts +++ b/test/integrations/destinations/braze/dataDelivery/data.ts @@ -544,32 +544,96 @@ export const data = [ status: 200, body: { output: { - status: 201, - message: 'Request for braze Processed Successfully', - destinationResponse: { - response: { - aliases_processed: 1, - message: 'success', + response: [ + { + error: '{"aliases_processed":1,"message":"success"}', + statusCode: 201, + metadata: { + jobId: 2, + attemptNum: 0, + userId: '', + sourceId: '2Vsge2uWYdrLfG7pZb5Y82eo4lr', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', + secret: { + access_token: 'secret', + refresh_token: 'refresh', + developer_token: 'developer_Token', + }, + }, }, - status: 201, - rudderJobMetadata: { - jobId: 2, - attemptNum: 0, - userId: '', - sourceId: '2Vsge2uWYdrLfG7pZb5Y82eo4lr', - destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', - workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', - secret: { - access_token: 'secret', - refresh_token: 'refresh', - developer_token: 'developer_Token', + ], + }, + }, + }, + }, + }, + { + name: 'braze', + description: 'Test Transformer Proxy V1 input with v0 proxy handler Error returned', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/testV1', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ + { + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + metadata: [ + { + jobId: 2, + attemptNum: 0, + userId: '', + sourceId: '2Vsge2uWYdrLfG7pZb5Y82eo4lr', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', + secret: { + access_token: 'secret', + refresh_token: 'refresh', + developer_token: 'developer_Token', }, }, + ], + files: {}, + params: { + destination: 'braze', + }, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { response: [ { - error: '{"aliases_processed":1,"message":"success"}', - statusCode: 201, + error: 'Request failed for braze with status: 401', + statusCode: 401, metadata: { jobId: 2, attemptNum: 0, @@ -585,6 +649,16 @@ export const data = [ }, }, ], + statTags: { + destType: 'BRAZE', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + errorCategory: 'network', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', + }, }, }, }, diff --git a/test/integrations/destinations/braze/network.ts b/test/integrations/destinations/braze/network.ts index 1c39a247e4..0ef99b806d 100644 --- a/test/integrations/destinations/braze/network.ts +++ b/test/integrations/destinations/braze/network.ts @@ -391,5 +391,34 @@ const deleteNwData = [ }, }, }, + { + httpReq: { + url: 'https://rest.iad-03.braze.com/users/identify/testV1', + data: { + aliases_to_identify: [ + { + external_id: 'gabi_userId_45', + user_alias: { alias_label: 'rudder_id', alias_name: 'gabi_anonId_45' }, + }, + ], + }, + params: { destination: 'braze' }, + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + 'User-Agent': 'RudderLabs', + }, + method: 'POST', + }, + httpRes: { + data: { + code: 400, + message: 'Bad Req', + status: 'Fail Case', + }, + status: 401, + }, + }, ]; export const networkCallsData = [...deleteNwData, ...dataDeliveryMocksData]; From 55836a29e08b4253aa00fc9a2b298fb177253cc4 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 11 Dec 2023 11:37:29 +0530 Subject: [PATCH 15/15] fix: updated DeliveriesResponse type --- src/types/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/types/index.ts b/src/types/index.ts index f4432e5c2a..61590d0b27 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -187,8 +187,6 @@ type DeliveryJobState = { }; type DeliveriesResponse = { - status?: number; - message?: string; statTags?: object; authErrorCategory?: string; response: DeliveryJobState[];