From 722f3d1a61c75278c01294d55d1f5c06e010bc81 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Wed, 24 Apr 2024 18:10:02 +0530 Subject: [PATCH] feat: move hubspot to transformer proxy to enable partial batch handling --- src/v1/destinations/hs/networkHandler.js | 101 +++++++ .../destinations/hs/dataDelivery/business.ts | 261 ++++++++++++++++++ .../destinations/hs/dataDelivery/data.ts | 4 + .../destinations/hs/dataDelivery/other.ts | 233 ++++++++++++++++ test/integrations/destinations/hs/network.ts | 168 +++++++++++ 5 files changed, 767 insertions(+) create mode 100644 src/v1/destinations/hs/networkHandler.js create mode 100644 test/integrations/destinations/hs/dataDelivery/business.ts create mode 100644 test/integrations/destinations/hs/dataDelivery/data.ts create mode 100644 test/integrations/destinations/hs/dataDelivery/other.ts diff --git a/src/v1/destinations/hs/networkHandler.js b/src/v1/destinations/hs/networkHandler.js new file mode 100644 index 0000000000..2a7e534b9e --- /dev/null +++ b/src/v1/destinations/hs/networkHandler.js @@ -0,0 +1,101 @@ +/* eslint-disable no-param-reassign */ +/* eslint-disable no-restricted-syntax */ +const { removeUndefinedAndNullValues } = require('@rudderstack/integrations-lib'); +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); +const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); +const { isHttpStatusSuccess, getAuthErrCategoryFromStCode } = require('../../../v0/util/index'); + +const { + processAxiosResponse, + getDynamicErrorType, +} = require('../../../adapters/utils/networkUtils'); +const tags = require('../../../v0/util/tags'); + +const populateErrorMessage = (response) => { + const errorResponse = { + message: response?.message || null, + category: response?.category || null, + correlationId: response?.correlationId || null, + }; + removeUndefinedAndNullValues(errorResponse); + if (Object.keys(errorResponse).length === 0) { + return 'unknown error format'; + } + return JSON.stringify(errorResponse); +}; + +const responseHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata } = responseParams; + const message = `[HUBSPOT Response V1 Handler] - Request Processed Successfully`; + const responseWithIndividualEvents = []; + const { response, status } = destinationResponse; + + if (isHttpStatusSuccess(status)) { + // populate different response for each event + const results = response?.results; + if (Array.isArray(results)) { + for (const [idx] of results.entries()) { + const proxyOutputObj = { + statusCode: 200, + metadata: rudderJobMetadata[idx], + error: 'success', + }; + responseWithIndividualEvents.push(proxyOutputObj); + } + } + + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; + } + + // in case of failure status, populate response to maintain len(metadata)=len(response) + const errorMessage = populateErrorMessage(response); + + // At least one event in the batch is invalid. + if (status === 400 && rudderJobMetadata.length > 1) { + if (rudderJobMetadata.length > 1) { + for (const metadata of rudderJobMetadata) { + metadata.dontBatch = true; + responseWithIndividualEvents.push({ + statusCode: status, + metadata, + error: errorMessage, + }); + } + } + // sending back 500 for retry only when events came in a batch + throw new TransformerProxyError( + `HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation`, + 500, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(500), + }, + destinationResponse, + '', + responseWithIndividualEvents, + ); + } + throw new TransformerProxyError( + `HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation`, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + getAuthErrCategoryFromStCode(status), + responseWithIndividualEvents, + ); +}; + +function networkHandler() { + this.prepareProxy = prepareProxyRequest; + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.responseHandler = responseHandler; +} + +module.exports = { networkHandler }; diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts new file mode 100644 index 0000000000..24988dd6a5 --- /dev/null +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -0,0 +1,261 @@ +import { generateMetadata, generateProxyV1Payload } from '../../../testUtils'; + +const commonStatTags = { + destType: 'HS', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', +}; +export const businessData = [ + { + name: 'hs', + description: 'successfully creating users from a batch', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + JSON: { + inputs: [ + { + properties: { + firstname: 'testmail1217', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + [generateMetadata(1), generateMetadata(2)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: 'success', + metadata: generateMetadata(1), + statusCode: 200, + }, + { + error: 'success', + metadata: generateMetadata(2), + statusCode: 200, + }, + ], + destinationResponse: { + response: { + status: 'COMPLETE', + results: [ + { + id: '12877907025', + properties: { + createdate: '2024-04-16T09:50:16.034Z', + email: 'test1@mail.com', + firstname: 'test1', + hs_is_unworked: 'true', + hs_object_id: '12877907025', + hs_pipeline: 'contacts-lifecycle-pipeline', + lastmodifieddate: '2024-04-23T11:52:03.723Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-04-16T09:50:16.034Z', + updatedAt: '2024-04-23T11:52:03.723Z', + archived: false, + }, + { + id: '12877907024', + properties: { + createdate: '2024-04-16T09:50:16.034Z', + firstname: 'testmail1217', + hs_is_unworked: 'true', + hs_object_id: '12877907024', + hs_pipeline: 'contacts-lifecycle-pipeline', + lastmodifieddate: '2024-04-23T11:52:03.723Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-04-16T09:50:16.034Z', + updatedAt: '2024-04-23T11:52:03.723Z', + archived: false, + }, + ], + startedAt: '2024-04-24T05:11:51.090Z', + completedAt: '2024-04-24T05:11:51.190Z', + }, + status: 200, + }, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed due to duplicate object in a batch', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + JSON: { + inputs: [ + { + properties: { + firstname: 'test5', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + { + properties: { + firstname: 'testmail1217', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + [generateMetadata(1), generateMetadata(2)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"status":"error","message":"Duplicate IDs found in batch input: [12877907025]. IDs must be unique","correlationId":"d24ec5cd-8998-4674-a928-59603ae6b0eb","context":{"ids":["12877907025"]},"category":"VALIDATION_ERROR"}', + metadata: { + ...generateMetadata(1), + dontBatch: true, + }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Duplicate IDs found in batch input: [12877907025]. IDs must be unique","correlationId":"d24ec5cd-8998-4674-a928-59603ae6b0eb","context":{"ids":["12877907025"]},"category":"VALIDATION_ERROR"}', + metadata: { + ...generateMetadata(2), + dontBatch: true, + }, + statusCode: 500, + }, + ], + statTags: commonStatTags, + status: 500, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed due to wrong email format in a batch', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + JSON: { + inputs: [ + [ + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + }, + { + properties: { + firstname: 'testmail1217', + email: 'testmail1217@testmail.com', + }, + }, + { + properties: { + firstname: 'test5', + email: 'test5@xmail.con', + }, + }, + ], + ], + }, + }, + [generateMetadata(1), generateMetadata(2), generateMetadata(3)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"status":"error","message":"Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)","correlationId":"99df04b9-da11-4504-bd97-2c15f58d0943"}', + metadata: { + ...generateMetadata(1), + dontBatch: true, + }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)","correlationId":"99df04b9-da11-4504-bd97-2c15f58d0943"}', + metadata: { + ...generateMetadata(2), + dontBatch: true, + }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)","correlationId":"99df04b9-da11-4504-bd97-2c15f58d0943"}', + metadata: { + ...generateMetadata(3), + dontBatch: true, + }, + statusCode: 500, + }, + ], + statTags: commonStatTags, + status: 500, + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/hs/dataDelivery/data.ts b/test/integrations/destinations/hs/dataDelivery/data.ts new file mode 100644 index 0000000000..5b2060d001 --- /dev/null +++ b/test/integrations/destinations/hs/dataDelivery/data.ts @@ -0,0 +1,4 @@ +import { businessData } from './business'; +import { otherData } from './other'; + +export const data = [...businessData, ...otherData]; diff --git a/test/integrations/destinations/hs/dataDelivery/other.ts b/test/integrations/destinations/hs/dataDelivery/other.ts new file mode 100644 index 0000000000..9b1e5d5e80 --- /dev/null +++ b/test/integrations/destinations/hs/dataDelivery/other.ts @@ -0,0 +1,233 @@ +import { generateMetadata } from '../../../testUtils'; + +const commonStatTags = { + destType: 'HS', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', +}; + +const commonBody = { + version: '1', + type: 'REST', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummyAccessToken', + }, + params: {}, + files: {}, + metadata: [generateMetadata(1), generateMetadata(2)], + body: { + JSON: { + inputs: [ + { + properties: { + firstname: 'testmail1217', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, +}; + +export const otherData = [ + { + name: 'hs', + description: 'failed due to gateway timeout from hubspot', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://random_test_url/test_for_gateway_time_out', + ...commonBody, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: '"Gateway Timeout"', + metadata: generateMetadata(1), + statusCode: 504, + }, + { + error: '"Gateway Timeout"', + metadata: generateMetadata(2), + statusCode: 504, + }, + ], + statTags: commonStatTags, + status: 504, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed due to internal server error from hubspot', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://random_test_url/test_for_internal_server_error', + ...commonBody, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: '"Internal Server Error"', + metadata: generateMetadata(1), + statusCode: 500, + }, + { + error: '"Internal Server Error"', + metadata: generateMetadata(2), + statusCode: 500, + }, + ], + statTags: commonStatTags, + status: 500, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed due to service unavailable error from hubspot', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://random_test_url/test_for_service_not_available', + ...commonBody, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}', + metadata: generateMetadata(1), + statusCode: 503, + }, + { + error: + '{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}', + metadata: generateMetadata(2), + statusCode: 503, + }, + ], + statTags: commonStatTags, + status: 503, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'getting success response but not in the expected format', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + version: '1', + type: 'REST', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummyAccessToken', + }, + params: {}, + files: {}, + metadata: [generateMetadata(1), generateMetadata(2)], + body: { + JSON: { + inputs: [ + { + properties: { + firstname: 'testmail12178', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [], + status: 200, + destinationResponse: { + status: 200, + response: { + message: 'unknown response', + }, + }, + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/hs/network.ts b/test/integrations/destinations/hs/network.ts index 3d3b8fd83f..cfc24d5498 100644 --- a/test/integrations/destinations/hs/network.ts +++ b/test/integrations/destinations/hs/network.ts @@ -692,4 +692,172 @@ export const networkCallsData = [ status: 200, }, }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + { + properties: { + firstname: 'testmail1217', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + httpRes: { + status: 200, + data: { + status: 'COMPLETE', + results: [ + { + id: '12877907025', + properties: { + createdate: '2024-04-16T09:50:16.034Z', + email: 'test1@mail.com', + firstname: 'test1', + hs_is_unworked: 'true', + hs_object_id: '12877907025', + hs_pipeline: 'contacts-lifecycle-pipeline', + lastmodifieddate: '2024-04-23T11:52:03.723Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-04-16T09:50:16.034Z', + updatedAt: '2024-04-23T11:52:03.723Z', + archived: false, + }, + { + id: '12877907024', + properties: { + createdate: '2024-04-16T09:50:16.034Z', + firstname: 'testmail1217', + hs_is_unworked: 'true', + hs_object_id: '12877907024', + hs_pipeline: 'contacts-lifecycle-pipeline', + lastmodifieddate: '2024-04-23T11:52:03.723Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-04-16T09:50:16.034Z', + updatedAt: '2024-04-23T11:52:03.723Z', + archived: false, + }, + ], + startedAt: '2024-04-24T05:11:51.090Z', + completedAt: '2024-04-24T05:11:51.190Z', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + { + properties: { + firstname: 'test5', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + { + properties: { + firstname: 'testmail1217', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + httpRes: { + status: 400, + data: { + status: 'error', + message: 'Duplicate IDs found in batch input: [12877907025]. IDs must be unique', + correlationId: 'd24ec5cd-8998-4674-a928-59603ae6b0eb', + context: { + ids: ['12877907025'], + }, + category: 'VALIDATION_ERROR', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + [ + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + }, + { + properties: { + firstname: 'testmail1217', + email: 'testmail1217@testmail.com', + }, + }, + { + properties: { + firstname: 'test5', + email: 'test5@xmail.con', + }, + }, + ], + ], + }, + }, + httpRes: { + status: 400, + data: { + status: 'error', + message: + 'Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)', + correlationId: '99df04b9-da11-4504-bd97-2c15f58d0943', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + { + properties: { + firstname: 'testmail12178', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + httpRes: { + status: 200, + data: { + message: 'unknown response', + }, + }, + }, ];