From 722f3d1a61c75278c01294d55d1f5c06e010bc81 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Wed, 24 Apr 2024 18:10:02 +0530 Subject: [PATCH 01/11] feat: move hubspot to transformer proxy to enable partial batch handling --- src/v1/destinations/hs/networkHandler.js | 101 +++++++ .../destinations/hs/dataDelivery/business.ts | 261 ++++++++++++++++++ .../destinations/hs/dataDelivery/data.ts | 4 + .../destinations/hs/dataDelivery/other.ts | 233 ++++++++++++++++ test/integrations/destinations/hs/network.ts | 168 +++++++++++ 5 files changed, 767 insertions(+) create mode 100644 src/v1/destinations/hs/networkHandler.js create mode 100644 test/integrations/destinations/hs/dataDelivery/business.ts create mode 100644 test/integrations/destinations/hs/dataDelivery/data.ts create mode 100644 test/integrations/destinations/hs/dataDelivery/other.ts diff --git a/src/v1/destinations/hs/networkHandler.js b/src/v1/destinations/hs/networkHandler.js new file mode 100644 index 0000000000..2a7e534b9e --- /dev/null +++ b/src/v1/destinations/hs/networkHandler.js @@ -0,0 +1,101 @@ +/* eslint-disable no-param-reassign */ +/* eslint-disable no-restricted-syntax */ +const { removeUndefinedAndNullValues } = require('@rudderstack/integrations-lib'); +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); +const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); +const { isHttpStatusSuccess, getAuthErrCategoryFromStCode } = require('../../../v0/util/index'); + +const { + processAxiosResponse, + getDynamicErrorType, +} = require('../../../adapters/utils/networkUtils'); +const tags = require('../../../v0/util/tags'); + +const populateErrorMessage = (response) => { + const errorResponse = { + message: response?.message || null, + category: response?.category || null, + correlationId: response?.correlationId || null, + }; + removeUndefinedAndNullValues(errorResponse); + if (Object.keys(errorResponse).length === 0) { + return 'unknown error format'; + } + return JSON.stringify(errorResponse); +}; + +const responseHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata } = responseParams; + const message = `[HUBSPOT Response V1 Handler] - Request Processed Successfully`; + const responseWithIndividualEvents = []; + const { response, status } = destinationResponse; + + if (isHttpStatusSuccess(status)) { + // populate different response for each event + const results = response?.results; + if (Array.isArray(results)) { + for (const [idx] of results.entries()) { + const proxyOutputObj = { + statusCode: 200, + metadata: rudderJobMetadata[idx], + error: 'success', + }; + responseWithIndividualEvents.push(proxyOutputObj); + } + } + + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; + } + + // in case of failure status, populate response to maintain len(metadata)=len(response) + const errorMessage = populateErrorMessage(response); + + // At least one event in the batch is invalid. + if (status === 400 && rudderJobMetadata.length > 1) { + if (rudderJobMetadata.length > 1) { + for (const metadata of rudderJobMetadata) { + metadata.dontBatch = true; + responseWithIndividualEvents.push({ + statusCode: status, + metadata, + error: errorMessage, + }); + } + } + // sending back 500 for retry only when events came in a batch + throw new TransformerProxyError( + `HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation`, + 500, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(500), + }, + destinationResponse, + '', + responseWithIndividualEvents, + ); + } + throw new TransformerProxyError( + `HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation`, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + getAuthErrCategoryFromStCode(status), + responseWithIndividualEvents, + ); +}; + +function networkHandler() { + this.prepareProxy = prepareProxyRequest; + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.responseHandler = responseHandler; +} + +module.exports = { networkHandler }; diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts new file mode 100644 index 0000000000..24988dd6a5 --- /dev/null +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -0,0 +1,261 @@ +import { generateMetadata, generateProxyV1Payload } from '../../../testUtils'; + +const commonStatTags = { + destType: 'HS', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', +}; +export const businessData = [ + { + name: 'hs', + description: 'successfully creating users from a batch', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + JSON: { + inputs: [ + { + properties: { + firstname: 'testmail1217', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + [generateMetadata(1), generateMetadata(2)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: 'success', + metadata: generateMetadata(1), + statusCode: 200, + }, + { + error: 'success', + metadata: generateMetadata(2), + statusCode: 200, + }, + ], + destinationResponse: { + response: { + status: 'COMPLETE', + results: [ + { + id: '12877907025', + properties: { + createdate: '2024-04-16T09:50:16.034Z', + email: 'test1@mail.com', + firstname: 'test1', + hs_is_unworked: 'true', + hs_object_id: '12877907025', + hs_pipeline: 'contacts-lifecycle-pipeline', + lastmodifieddate: '2024-04-23T11:52:03.723Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-04-16T09:50:16.034Z', + updatedAt: '2024-04-23T11:52:03.723Z', + archived: false, + }, + { + id: '12877907024', + properties: { + createdate: '2024-04-16T09:50:16.034Z', + firstname: 'testmail1217', + hs_is_unworked: 'true', + hs_object_id: '12877907024', + hs_pipeline: 'contacts-lifecycle-pipeline', + lastmodifieddate: '2024-04-23T11:52:03.723Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-04-16T09:50:16.034Z', + updatedAt: '2024-04-23T11:52:03.723Z', + archived: false, + }, + ], + startedAt: '2024-04-24T05:11:51.090Z', + completedAt: '2024-04-24T05:11:51.190Z', + }, + status: 200, + }, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed due to duplicate object in a batch', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + JSON: { + inputs: [ + { + properties: { + firstname: 'test5', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + { + properties: { + firstname: 'testmail1217', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + [generateMetadata(1), generateMetadata(2)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"status":"error","message":"Duplicate IDs found in batch input: [12877907025]. IDs must be unique","correlationId":"d24ec5cd-8998-4674-a928-59603ae6b0eb","context":{"ids":["12877907025"]},"category":"VALIDATION_ERROR"}', + metadata: { + ...generateMetadata(1), + dontBatch: true, + }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Duplicate IDs found in batch input: [12877907025]. IDs must be unique","correlationId":"d24ec5cd-8998-4674-a928-59603ae6b0eb","context":{"ids":["12877907025"]},"category":"VALIDATION_ERROR"}', + metadata: { + ...generateMetadata(2), + dontBatch: true, + }, + statusCode: 500, + }, + ], + statTags: commonStatTags, + status: 500, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed due to wrong email format in a batch', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + JSON: { + inputs: [ + [ + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + }, + { + properties: { + firstname: 'testmail1217', + email: 'testmail1217@testmail.com', + }, + }, + { + properties: { + firstname: 'test5', + email: 'test5@xmail.con', + }, + }, + ], + ], + }, + }, + [generateMetadata(1), generateMetadata(2), generateMetadata(3)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"status":"error","message":"Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)","correlationId":"99df04b9-da11-4504-bd97-2c15f58d0943"}', + metadata: { + ...generateMetadata(1), + dontBatch: true, + }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)","correlationId":"99df04b9-da11-4504-bd97-2c15f58d0943"}', + metadata: { + ...generateMetadata(2), + dontBatch: true, + }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)","correlationId":"99df04b9-da11-4504-bd97-2c15f58d0943"}', + metadata: { + ...generateMetadata(3), + dontBatch: true, + }, + statusCode: 500, + }, + ], + statTags: commonStatTags, + status: 500, + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/hs/dataDelivery/data.ts b/test/integrations/destinations/hs/dataDelivery/data.ts new file mode 100644 index 0000000000..5b2060d001 --- /dev/null +++ b/test/integrations/destinations/hs/dataDelivery/data.ts @@ -0,0 +1,4 @@ +import { businessData } from './business'; +import { otherData } from './other'; + +export const data = [...businessData, ...otherData]; diff --git a/test/integrations/destinations/hs/dataDelivery/other.ts b/test/integrations/destinations/hs/dataDelivery/other.ts new file mode 100644 index 0000000000..9b1e5d5e80 --- /dev/null +++ b/test/integrations/destinations/hs/dataDelivery/other.ts @@ -0,0 +1,233 @@ +import { generateMetadata } from '../../../testUtils'; + +const commonStatTags = { + destType: 'HS', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', +}; + +const commonBody = { + version: '1', + type: 'REST', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummyAccessToken', + }, + params: {}, + files: {}, + metadata: [generateMetadata(1), generateMetadata(2)], + body: { + JSON: { + inputs: [ + { + properties: { + firstname: 'testmail1217', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, +}; + +export const otherData = [ + { + name: 'hs', + description: 'failed due to gateway timeout from hubspot', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://random_test_url/test_for_gateway_time_out', + ...commonBody, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: '"Gateway Timeout"', + metadata: generateMetadata(1), + statusCode: 504, + }, + { + error: '"Gateway Timeout"', + metadata: generateMetadata(2), + statusCode: 504, + }, + ], + statTags: commonStatTags, + status: 504, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed due to internal server error from hubspot', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://random_test_url/test_for_internal_server_error', + ...commonBody, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: '"Internal Server Error"', + metadata: generateMetadata(1), + statusCode: 500, + }, + { + error: '"Internal Server Error"', + metadata: generateMetadata(2), + statusCode: 500, + }, + ], + statTags: commonStatTags, + status: 500, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed due to service unavailable error from hubspot', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://random_test_url/test_for_service_not_available', + ...commonBody, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}', + metadata: generateMetadata(1), + statusCode: 503, + }, + { + error: + '{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}', + metadata: generateMetadata(2), + statusCode: 503, + }, + ], + statTags: commonStatTags, + status: 503, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'getting success response but not in the expected format', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + version: '1', + type: 'REST', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummyAccessToken', + }, + params: {}, + files: {}, + metadata: [generateMetadata(1), generateMetadata(2)], + body: { + JSON: { + inputs: [ + { + properties: { + firstname: 'testmail12178', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [], + status: 200, + destinationResponse: { + status: 200, + response: { + message: 'unknown response', + }, + }, + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/hs/network.ts b/test/integrations/destinations/hs/network.ts index 3d3b8fd83f..cfc24d5498 100644 --- a/test/integrations/destinations/hs/network.ts +++ b/test/integrations/destinations/hs/network.ts @@ -692,4 +692,172 @@ export const networkCallsData = [ status: 200, }, }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + { + properties: { + firstname: 'testmail1217', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + httpRes: { + status: 200, + data: { + status: 'COMPLETE', + results: [ + { + id: '12877907025', + properties: { + createdate: '2024-04-16T09:50:16.034Z', + email: 'test1@mail.com', + firstname: 'test1', + hs_is_unworked: 'true', + hs_object_id: '12877907025', + hs_pipeline: 'contacts-lifecycle-pipeline', + lastmodifieddate: '2024-04-23T11:52:03.723Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-04-16T09:50:16.034Z', + updatedAt: '2024-04-23T11:52:03.723Z', + archived: false, + }, + { + id: '12877907024', + properties: { + createdate: '2024-04-16T09:50:16.034Z', + firstname: 'testmail1217', + hs_is_unworked: 'true', + hs_object_id: '12877907024', + hs_pipeline: 'contacts-lifecycle-pipeline', + lastmodifieddate: '2024-04-23T11:52:03.723Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-04-16T09:50:16.034Z', + updatedAt: '2024-04-23T11:52:03.723Z', + archived: false, + }, + ], + startedAt: '2024-04-24T05:11:51.090Z', + completedAt: '2024-04-24T05:11:51.190Z', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + { + properties: { + firstname: 'test5', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + { + properties: { + firstname: 'testmail1217', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + httpRes: { + status: 400, + data: { + status: 'error', + message: 'Duplicate IDs found in batch input: [12877907025]. IDs must be unique', + correlationId: 'd24ec5cd-8998-4674-a928-59603ae6b0eb', + context: { + ids: ['12877907025'], + }, + category: 'VALIDATION_ERROR', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + [ + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + }, + { + properties: { + firstname: 'testmail1217', + email: 'testmail1217@testmail.com', + }, + }, + { + properties: { + firstname: 'test5', + email: 'test5@xmail.con', + }, + }, + ], + ], + }, + }, + httpRes: { + status: 400, + data: { + status: 'error', + message: + 'Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)', + correlationId: '99df04b9-da11-4504-bd97-2c15f58d0943', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + { + properties: { + firstname: 'testmail12178', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + httpRes: { + status: 200, + data: { + message: 'unknown response', + }, + }, + }, ]; From effc2e173ded844275cb5b9d3271b3977a7c8402 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Wed, 22 May 2024 19:28:08 +0530 Subject: [PATCH 02/11] chore: refactor code --- src/v1/destinations/hs/networkHandler.js | 75 +++++++++---------- .../destinations/hs/dataDelivery/business.ts | 34 +++++++-- .../destinations/hs/dataDelivery/other.ts | 15 +++- 3 files changed, 73 insertions(+), 51 deletions(-) diff --git a/src/v1/destinations/hs/networkHandler.js b/src/v1/destinations/hs/networkHandler.js index 2a7e534b9e..ff694efc61 100644 --- a/src/v1/destinations/hs/networkHandler.js +++ b/src/v1/destinations/hs/networkHandler.js @@ -1,6 +1,5 @@ /* eslint-disable no-param-reassign */ /* eslint-disable no-restricted-syntax */ -const { removeUndefinedAndNullValues } = require('@rudderstack/integrations-lib'); const { TransformerProxyError } = require('../../../v0/util/errorTypes'); const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); const { isHttpStatusSuccess, getAuthErrCategoryFromStCode } = require('../../../v0/util/index'); @@ -11,30 +10,35 @@ const { } = require('../../../adapters/utils/networkUtils'); const tags = require('../../../v0/util/tags'); -const populateErrorMessage = (response) => { - const errorResponse = { - message: response?.message || null, - category: response?.category || null, - correlationId: response?.correlationId || null, - }; - removeUndefinedAndNullValues(errorResponse); - if (Object.keys(errorResponse).length === 0) { - return 'unknown error format'; +const verify = (results, rudderJobMetadata) => + Array.isArray(results) && results.length === rudderJobMetadata.length; + +const populateResponseWithDontBatch = (rudderJobMetadata, status, response) => { + const errorMessage = JSON.stringify(response); + const responseWithIndividualEvents = []; + for (const metadata of rudderJobMetadata) { + metadata.dontBatch = true; + responseWithIndividualEvents.push({ + statusCode: status, + metadata, + error: errorMessage, + }); } - return JSON.stringify(errorResponse); + return responseWithIndividualEvents; }; - const responseHandler = (responseParams) => { const { destinationResponse, rudderJobMetadata } = responseParams; - const message = `[HUBSPOT Response V1 Handler] - Request Processed Successfully`; + const successMessage = `[HUBSPOT Response V1 Handler] - Request Processed Successfully`; + const failureMessage = + 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation'; const responseWithIndividualEvents = []; const { response, status } = destinationResponse; if (isHttpStatusSuccess(status)) { // populate different response for each event const results = response?.results; - if (Array.isArray(results)) { - for (const [idx] of results.entries()) { + if (verify(results, rudderJobMetadata)) { + for (const [idx] of rudderJobMetadata.entries()) { const proxyOutputObj = { statusCode: 200, metadata: rudderJobMetadata[idx], @@ -42,45 +46,34 @@ const responseHandler = (responseParams) => { }; responseWithIndividualEvents.push(proxyOutputObj); } + return { + status, + message: successMessage, + destinationResponse, + response: responseWithIndividualEvents, + }; } - + // return the destiantionResponse as it is when the response is not in expected format return { status, - message, + message: successMessage, destinationResponse, - response: responseWithIndividualEvents, + response: destinationResponse, }; } - // in case of failure status, populate response to maintain len(metadata)=len(response) - const errorMessage = populateErrorMessage(response); - // At least one event in the batch is invalid. if (status === 400 && rudderJobMetadata.length > 1) { - if (rudderJobMetadata.length > 1) { - for (const metadata of rudderJobMetadata) { - metadata.dontBatch = true; - responseWithIndividualEvents.push({ - statusCode: status, - metadata, - error: errorMessage, - }); - } - } // sending back 500 for retry only when events came in a batch - throw new TransformerProxyError( - `HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation`, - 500, - { - [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(500), - }, + return { + status: 500, + message: failureMessage, destinationResponse, - '', - responseWithIndividualEvents, - ); + response: populateResponseWithDontBatch(rudderJobMetadata, status, response), + }; } throw new TransformerProxyError( - `HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation`, + failureMessage, status, { [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts index 24988dd6a5..e4d8abdecc 100644 --- a/test/integrations/destinations/hs/dataDelivery/business.ts +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -112,6 +112,7 @@ export const businessData = [ { name: 'hs', description: 'failed due to duplicate object in a batch', + id: 'hs_datadelivery_01', feature: 'dataDelivery', module: 'destination', version: 'v1', @@ -148,6 +149,18 @@ export const businessData = [ status: 200, body: { output: { + destinationResponse: { + response: { + category: 'VALIDATION_ERROR', + context: { + ids: ['12877907025'], + }, + correlationId: 'd24ec5cd-8998-4674-a928-59603ae6b0eb', + message: 'Duplicate IDs found in batch input: [12877907025]. IDs must be unique', + status: 'error', + }, + status: 400, + }, message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', response: [ { @@ -157,7 +170,7 @@ export const businessData = [ ...generateMetadata(1), dontBatch: true, }, - statusCode: 500, + statusCode: 400, }, { error: @@ -166,10 +179,9 @@ export const businessData = [ ...generateMetadata(2), dontBatch: true, }, - statusCode: 500, + statusCode: 400, }, ], - statTags: commonStatTags, status: 500, }, }, @@ -221,6 +233,15 @@ export const businessData = [ status: 200, body: { output: { + destinationResponse: { + response: { + correlationId: '99df04b9-da11-4504-bd97-2c15f58d0943', + message: + 'Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)', + status: 'error', + }, + status: 400, + }, message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', response: [ { @@ -230,7 +251,7 @@ export const businessData = [ ...generateMetadata(1), dontBatch: true, }, - statusCode: 500, + statusCode: 400, }, { error: @@ -239,7 +260,7 @@ export const businessData = [ ...generateMetadata(2), dontBatch: true, }, - statusCode: 500, + statusCode: 400, }, { error: @@ -248,10 +269,9 @@ export const businessData = [ ...generateMetadata(3), dontBatch: true, }, - statusCode: 500, + statusCode: 400, }, ], - statTags: commonStatTags, status: 500, }, }, diff --git a/test/integrations/destinations/hs/dataDelivery/other.ts b/test/integrations/destinations/hs/dataDelivery/other.ts index 9b1e5d5e80..696cd16d30 100644 --- a/test/integrations/destinations/hs/dataDelivery/other.ts +++ b/test/integrations/destinations/hs/dataDelivery/other.ts @@ -49,6 +49,7 @@ const commonBody = { export const otherData = [ { name: 'hs', + id: 'hs_datadelivery_other_00', description: 'failed due to gateway timeout from hubspot', feature: 'dataDelivery', module: 'destination', @@ -88,6 +89,7 @@ export const otherData = [ }, { name: 'hs', + id: 'hs_datadelivery_other_01', description: 'failed due to internal server error from hubspot', feature: 'dataDelivery', module: 'destination', @@ -127,6 +129,7 @@ export const otherData = [ }, { name: 'hs', + id: 'hs_datadelivery_other_02', description: 'failed due to service unavailable error from hubspot', feature: 'dataDelivery', module: 'destination', @@ -168,6 +171,7 @@ export const otherData = [ }, { name: 'hs', + id: 'hs_datadelivery_other_03', description: 'getting success response but not in the expected format', feature: 'dataDelivery', module: 'destination', @@ -216,15 +220,20 @@ export const otherData = [ status: 200, body: { output: { - message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', - response: [], - status: 200, destinationResponse: { + response: { + message: 'unknown response', + }, status: 200, + }, + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: { response: { message: 'unknown response', }, + status: 200, }, + status: 200, }, }, }, From f62271027ea89de61d6ef9ae3ed1fb9ea874bd21 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Fri, 24 May 2024 08:56:23 +0530 Subject: [PATCH 03/11] chore: add unit tests --- src/v0/destinations/hs/transform.js | 33 +- src/v0/destinations/hs/util.js | 32 + src/v1/destinations/hs/networkHandler.js | 2 +- .../destinations/hs/dataDelivery/business.ts | 10 +- test/integrations/destinations/hs/network.ts | 16 + .../destinations/hs/router/data.ts | 931 ++++++++++++++++++ 6 files changed, 1015 insertions(+), 9 deletions(-) diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js index 9eed244af4..e50f7cbfa3 100644 --- a/src/v0/destinations/hs/transform.js +++ b/src/v0/destinations/hs/transform.js @@ -1,5 +1,6 @@ const get = require('get-value'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { groupBy } = require('lodash'); const { EventType } = require('../../../constants'); const { handleRtTfSingleEventError, getDestinationExternalIDInfoForRetl } = require('../../util'); const { API_VERSION } = require('./config'); @@ -15,6 +16,7 @@ const { fetchFinalSetOfTraits, getProperties, validateDestinationConfig, + convertToResponseFormat, } = require('./util'); const processSingleMessage = async (message, destination, propertyMap) => { @@ -137,16 +139,41 @@ const processRouterDest = async (inputs, reqMetadata) => { }), ); + const groupedByDifferentDontBatch = groupBy( + successRespList, + (response) => response.metadata.dontBatch, + ); + + const dontBatchTrueResponses = []; + const dontBatchFalseOrUndefinedResponses = []; + Object.keys(groupedByDifferentDontBatch).forEach((dontaBatchVal) => { + switch (dontaBatchVal) { + case 'true': + dontBatchTrueResponses.push(...groupedByDifferentDontBatch.true); + break; + case 'false': + dontBatchFalseOrUndefinedResponses.push(...groupedByDifferentDontBatch.false); + break; + case 'undefined': + dontBatchFalseOrUndefinedResponses.push(...groupedByDifferentDontBatch.undefined); + break; + default: + } + }); // batch implementation let batchedResponseList = []; if (successRespList.length > 0) { if (destination.Config.apiVersion === API_VERSION.v3) { - batchedResponseList = batchEvents(successRespList); + batchedResponseList = batchEvents(dontBatchFalseOrUndefinedResponses); } else { - batchedResponseList = legacyBatchEvents(successRespList); + batchedResponseList = legacyBatchEvents(dontBatchFalseOrUndefinedResponses); } } - return [...batchedResponseList, ...errorRespList]; + return [ + ...batchedResponseList, + ...errorRespList, + ...convertToResponseFormat(dontBatchTrueResponses), + ]; }; module.exports = { process, processRouterDest }; diff --git a/src/v0/destinations/hs/util.js b/src/v0/destinations/hs/util.js index b30207fe15..75ec073725 100644 --- a/src/v0/destinations/hs/util.js +++ b/src/v0/destinations/hs/util.js @@ -22,6 +22,9 @@ const { getValueFromMessage, isNull, validateEventName, + defaultBatchRequestConfig, + defaultPostRequestConfig, + getSuccessRespEvents, } = require('../../util'); const { CONTACT_PROPERTY_MAP_ENDPOINT, @@ -837,6 +840,34 @@ const addExternalIdToHSTraits = (message) => { set(getFieldValueFromMessage(message, 'traits'), externalIdObj.identifierType, externalIdObj.id); }; +const convertToResponseFormat = (successRespListWithDontBatchTrue) => { + const response = []; + if (Array.isArray(successRespListWithDontBatchTrue)) { + successRespListWithDontBatchTrue.forEach((event) => { + const { message, metadata, destination } = event; + const endpoint = get(message, 'endpoint'); + + const batchedResponse = defaultBatchRequestConfig(); + batchedResponse.batchedRequest.headers = message.headers; + batchedResponse.batchedRequest.endpoint = endpoint; + batchedResponse.batchedRequest.body = message.body; + batchedResponse.batchedRequest.params = message.params; + batchedResponse.batchedRequest.method = defaultPostRequestConfig.requestMethod; + batchedResponse.metadata = [metadata]; + batchedResponse.destination = destination; + + response.push( + getSuccessRespEvents( + batchedResponse.batchedRequest, + batchedResponse.metadata, + batchedResponse.destination, + ), + ); + }); + } + return response; +}; + module.exports = { validateDestinationConfig, addExternalIdToHSTraits, @@ -856,4 +887,5 @@ module.exports = { getObjectAndIdentifierType, extractIDsForSearchAPI, getRequestData, + convertToResponseFormat, }; diff --git a/src/v1/destinations/hs/networkHandler.js b/src/v1/destinations/hs/networkHandler.js index ff694efc61..5caa8be76f 100644 --- a/src/v1/destinations/hs/networkHandler.js +++ b/src/v1/destinations/hs/networkHandler.js @@ -19,7 +19,7 @@ const populateResponseWithDontBatch = (rudderJobMetadata, status, response) => { for (const metadata of rudderJobMetadata) { metadata.dontBatch = true; responseWithIndividualEvents.push({ - statusCode: status, + statusCode: 500, metadata, error: errorMessage, }); diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts index e4d8abdecc..e9c6b80c89 100644 --- a/test/integrations/destinations/hs/dataDelivery/business.ts +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -170,7 +170,7 @@ export const businessData = [ ...generateMetadata(1), dontBatch: true, }, - statusCode: 400, + statusCode: 500, }, { error: @@ -179,7 +179,7 @@ export const businessData = [ ...generateMetadata(2), dontBatch: true, }, - statusCode: 400, + statusCode: 500, }, ], status: 500, @@ -251,7 +251,7 @@ export const businessData = [ ...generateMetadata(1), dontBatch: true, }, - statusCode: 400, + statusCode: 500, }, { error: @@ -260,7 +260,7 @@ export const businessData = [ ...generateMetadata(2), dontBatch: true, }, - statusCode: 400, + statusCode: 500, }, { error: @@ -269,7 +269,7 @@ export const businessData = [ ...generateMetadata(3), dontBatch: true, }, - statusCode: 400, + statusCode: 500, }, ], status: 500, diff --git a/test/integrations/destinations/hs/network.ts b/test/integrations/destinations/hs/network.ts index cfc24d5498..a306c20e66 100644 --- a/test/integrations/destinations/hs/network.ts +++ b/test/integrations/destinations/hs/network.ts @@ -860,4 +860,20 @@ export const networkCallsData = [ }, }, }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/search', + method: 'POST', + headers: { + Authorization: 'Bearer dontbatchtrueaccesstoken', + }, + }, + httpRes: { + data: { + total: 0, + results: [], + }, + status: 200, + }, + }, ]; diff --git a/test/integrations/destinations/hs/router/data.ts b/test/integrations/destinations/hs/router/data.ts index ab3ca8cba8..7d8c619b9d 100644 --- a/test/integrations/destinations/hs/router/data.ts +++ b/test/integrations/destinations/hs/router/data.ts @@ -1828,4 +1828,935 @@ export const data = [ }, }, }, + { + name: 'hs', + description: 'if dontBatch is true we are not going to create a batch out of those events', + feature: 'router', + module: 'destination', + version: 'v0', + scenario: 'buisness', + id: 'dontbatchtrue', + successCriteria: + 'should not create a batch with the events if that events contains dontBatch true', + input: { + request: { + body: { + input: [ + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.461+05:30', + userId: 'sample_user_id425', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow', + email: 'identify425@test.com', + phone: '9112340425', + lastname: 'Sparrow', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + timestamp: '2024-05-23T16:49:57.070+05:30', + receivedAt: '2024-05-23T16:49:57.071+05:30', + anonymousId: '8d872292709c6fbe', + }, + metadata: { + jobId: 1, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + dontBatch: true, + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.461+05:30', + userId: 'sample_user_id738', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow738', + email: 'identify425@test.con', + phone: '9112340738', + lastname: 'Sparrow738', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + timestamp: '2024-05-23T16:49:57.071+05:30', + anonymousId: '8d872292709c6fbe738', + }, + metadata: { + userId: '<<>>8d872292709c6fbe738<<>>sample_user_id738', + jobId: 2, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + workerAssignedTime: '2024-05-23T16:49:58.569269+05:30', + dontBatch: true, + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.462+05:30', + userId: 'sample_user_id803', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow803', + email: 'identify803@test.com', + phone: '9112340803', + lastname: 'Sparrow803', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + anonymousId: '8d872292709c6fbe803', + originalTimestamp: '2024-05-23T16:49:57.462+05:30', + }, + metadata: { + userId: '<<>>8d872292709c6fbe803<<>>sample_user_id803', + jobId: 3, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.462+05:30', + userId: 'sample_user_id804', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow804', + email: 'identify804@test.con', + phone: '9112340804', + lastname: 'Sparrow804', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + anonymousId: '8d872292709c6fbe804', + originalTimestamp: '2024-05-23T16:49:57.462+05:30', + }, + metadata: { + userId: '<<>>8d872292709c6fbe804<<>>sample_user_id804', + jobId: 4, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + dontBatch: false, + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + ], + destType: 'hs', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: true, + batchedRequest: { + body: { + FORM: {}, + JSON: { + inputs: [ + { + properties: { + email: 'identify803@test.com', + firstname: 'John', + lastname: 'Sparrow803', + phone: '9112340803', + }, + }, + { + properties: { + email: 'identify804@test.con', + firstname: 'John', + lastname: 'Sparrow804', + phone: '9112340804', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/create', + files: {}, + headers: { + Authorization: 'Bearer dontbatchtrueaccesstoken', + 'Content-Type': 'application/json', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: { + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + IsProcessorEnabled: true, + Name: 'hs-1', + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + Transformations: [], + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + metadata: [ + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + jobId: 3, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + userId: '<<>>8d872292709c6fbe803<<>>sample_user_id803', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + dontBatch: false, + jobId: 4, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + userId: '<<>>8d872292709c6fbe804<<>>sample_user_id804', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + ], + statusCode: 200, + }, + { + batched: false, + batchedRequest: { + body: { + FORM: {}, + JSON: { + properties: { + email: 'identify425@test.com', + firstname: 'John', + lastname: 'Sparrow', + phone: '9112340425', + }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts', + files: {}, + headers: { + Authorization: 'Bearer dontbatchtrueaccesstoken', + 'Content-Type': 'application/json', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: { + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + IsProcessorEnabled: true, + Name: 'hs-1', + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + Transformations: [], + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + metadata: [ + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + dontBatch: true, + jobId: 1, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + ], + statusCode: 200, + }, + { + batched: false, + batchedRequest: { + body: { + FORM: {}, + JSON: { + properties: { + email: 'identify425@test.con', + firstname: 'John', + lastname: 'Sparrow738', + phone: '9112340738', + }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts', + files: {}, + headers: { + Authorization: 'Bearer dontbatchtrueaccesstoken', + 'Content-Type': 'application/json', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: { + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + IsProcessorEnabled: true, + Name: 'hs-1', + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + Transformations: [], + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + metadata: [ + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + dontBatch: true, + jobId: 2, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + userId: '<<>>8d872292709c6fbe738<<>>sample_user_id738', + workerAssignedTime: '2024-05-23T16:49:58.569269+05:30', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + ], + statusCode: 200, + }, + ], + }, + }, + }, + }, + { + name: 'hs', + description: 'if dontBatch is not available we are considering those as dontbatch false', + feature: 'router', + module: 'destination', + version: 'v0', + scenario: 'buisness', + id: 'dontbatchundefined', + successCriteria: 'all events should be batched', + input: { + request: { + body: { + input: [ + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.461+05:30', + userId: 'sample_user_id425', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow', + email: 'identify425@test.com', + phone: '9112340425', + lastname: 'Sparrow', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + timestamp: '2024-05-23T16:49:57.070+05:30', + receivedAt: '2024-05-23T16:49:57.071+05:30', + anonymousId: '8d872292709c6fbe', + }, + metadata: { + jobId: 1, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.461+05:30', + userId: 'sample_user_id738', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow738', + email: 'identify425@test.con', + phone: '9112340738', + lastname: 'Sparrow738', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + timestamp: '2024-05-23T16:49:57.071+05:30', + anonymousId: '8d872292709c6fbe738', + }, + metadata: { + userId: '<<>>8d872292709c6fbe738<<>>sample_user_id738', + jobId: 2, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + workerAssignedTime: '2024-05-23T16:49:58.569269+05:30', + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + ], + destType: 'hs', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: true, + batchedRequest: { + body: { + FORM: {}, + JSON: { + inputs: [ + { + properties: { + email: 'identify425@test.com', + firstname: 'John', + lastname: 'Sparrow', + phone: '9112340425', + }, + }, + { + properties: { + email: 'identify425@test.con', + firstname: 'John', + lastname: 'Sparrow738', + phone: '9112340738', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/create', + files: {}, + headers: { + Authorization: 'Bearer dontbatchtrueaccesstoken', + 'Content-Type': 'application/json', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: { + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + IsProcessorEnabled: true, + Name: 'hs-1', + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + Transformations: [], + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + metadata: [ + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + jobId: 1, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + jobId: 2, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + userId: '<<>>8d872292709c6fbe738<<>>sample_user_id738', + workerAssignedTime: '2024-05-23T16:49:58.569269+05:30', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + ], + statusCode: 200, + }, + ], + }, + }, + }, + }, ]; From 1b9c83b87c132e61b29e8ed7a9c48c8ffce8bb63 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Mon, 27 May 2024 08:54:03 +0530 Subject: [PATCH 04/11] chore: refactor code --- src/v0/destinations/hs/transform.js | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js index e50f7cbfa3..32efe7f71c 100644 --- a/src/v0/destinations/hs/transform.js +++ b/src/v0/destinations/hs/transform.js @@ -1,6 +1,5 @@ const get = require('get-value'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); -const { groupBy } = require('lodash'); const { EventType } = require('../../../constants'); const { handleRtTfSingleEventError, getDestinationExternalIDInfoForRetl } = require('../../util'); const { API_VERSION } = require('./config'); @@ -139,27 +138,17 @@ const processRouterDest = async (inputs, reqMetadata) => { }), ); - const groupedByDifferentDontBatch = groupBy( - successRespList, - (response) => response.metadata.dontBatch, - ); - const dontBatchTrueResponses = []; const dontBatchFalseOrUndefinedResponses = []; - Object.keys(groupedByDifferentDontBatch).forEach((dontaBatchVal) => { - switch (dontaBatchVal) { - case 'true': - dontBatchTrueResponses.push(...groupedByDifferentDontBatch.true); - break; - case 'false': - dontBatchFalseOrUndefinedResponses.push(...groupedByDifferentDontBatch.false); - break; - case 'undefined': - dontBatchFalseOrUndefinedResponses.push(...groupedByDifferentDontBatch.undefined); - break; - default: + // segregating successRepList depending on dontbatch value + successRespList.forEach((successResp) => { + if (successResp.metadata?.dontBatch) { + dontBatchTrueResponses.push(successResp); + } else { + dontBatchFalseOrUndefinedResponses.push(successResp); } }); + // batch implementation let batchedResponseList = []; if (successRespList.length > 0) { @@ -172,6 +161,7 @@ const processRouterDest = async (inputs, reqMetadata) => { return [ ...batchedResponseList, ...errorRespList, + // if there are any events where dontbatch set to true we need to update them according to the response format ...convertToResponseFormat(dontBatchTrueResponses), ]; }; From 0927f2a3214eb1800a16247ab247e734dd41eb07 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Mon, 27 May 2024 09:08:45 +0530 Subject: [PATCH 05/11] chore: update error message --- src/v1/destinations/hs/networkHandler.js | 2 +- .../destinations/hs/dataDelivery/business.ts | 6 ++++-- test/integrations/destinations/hs/dataDelivery/other.ts | 9 ++++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/v1/destinations/hs/networkHandler.js b/src/v1/destinations/hs/networkHandler.js index 5caa8be76f..90688584bf 100644 --- a/src/v1/destinations/hs/networkHandler.js +++ b/src/v1/destinations/hs/networkHandler.js @@ -30,7 +30,7 @@ const responseHandler = (responseParams) => { const { destinationResponse, rudderJobMetadata } = responseParams; const successMessage = `[HUBSPOT Response V1 Handler] - Request Processed Successfully`; const failureMessage = - 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation'; + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation'; const responseWithIndividualEvents = []; const { response, status } = destinationResponse; diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts index e9c6b80c89..2778ffbff7 100644 --- a/test/integrations/destinations/hs/dataDelivery/business.ts +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -161,7 +161,8 @@ export const businessData = [ }, status: 400, }, - message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', response: [ { error: @@ -242,7 +243,8 @@ export const businessData = [ }, status: 400, }, - message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', response: [ { error: diff --git a/test/integrations/destinations/hs/dataDelivery/other.ts b/test/integrations/destinations/hs/dataDelivery/other.ts index 696cd16d30..e99541802e 100644 --- a/test/integrations/destinations/hs/dataDelivery/other.ts +++ b/test/integrations/destinations/hs/dataDelivery/other.ts @@ -67,7 +67,8 @@ export const otherData = [ status: 200, body: { output: { - message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', response: [ { error: '"Gateway Timeout"', @@ -107,7 +108,8 @@ export const otherData = [ status: 200, body: { output: { - message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', response: [ { error: '"Internal Server Error"', @@ -147,7 +149,8 @@ export const otherData = [ status: 200, body: { output: { - message: 'HUBSPOT: Error transformer proxy v1 during HUBSPOT response transformation', + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', response: [ { error: From cded119d2041f3da164953ec9363183aaa048500 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Wed, 12 Jun 2024 09:54:30 +0530 Subject: [PATCH 06/11] fix: update network handler for legacy api --- src/v0/destinations/hs/transform.js | 2 +- src/v1/destinations/hs/networkHandler.js | 30 ++++++++++++------ .../destinations/hs/dataDelivery/business.ts | 3 ++ .../destinations/hs/dataDelivery/other.ts | 31 +++++++++++++------ 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js index 32efe7f71c..22c1888eea 100644 --- a/src/v0/destinations/hs/transform.js +++ b/src/v0/destinations/hs/transform.js @@ -151,7 +151,7 @@ const processRouterDest = async (inputs, reqMetadata) => { // batch implementation let batchedResponseList = []; - if (successRespList.length > 0) { + if (dontBatchFalseOrUndefinedResponses.length > 0) { if (destination.Config.apiVersion === API_VERSION.v3) { batchedResponseList = batchEvents(dontBatchFalseOrUndefinedResponses); } else { diff --git a/src/v1/destinations/hs/networkHandler.js b/src/v1/destinations/hs/networkHandler.js index 90688584bf..0e37a64327 100644 --- a/src/v1/destinations/hs/networkHandler.js +++ b/src/v1/destinations/hs/networkHandler.js @@ -10,8 +10,15 @@ const { } = require('../../../adapters/utils/networkUtils'); const tags = require('../../../v0/util/tags'); -const verify = (results, rudderJobMetadata) => - Array.isArray(results) && results.length === rudderJobMetadata.length; +const verify = (results, rudderJobMetadata, destinationConfig) => { + if (destinationConfig?.apiVersion === undefined) { + return false; + } + if (destinationConfig?.apiVersion === 'newApi') { + return Array.isArray(results) && results.length === rudderJobMetadata.length; + } + return true; +}; const populateResponseWithDontBatch = (rudderJobMetadata, status, response) => { const errorMessage = JSON.stringify(response); @@ -27,7 +34,7 @@ const populateResponseWithDontBatch = (rudderJobMetadata, status, response) => { return responseWithIndividualEvents; }; const responseHandler = (responseParams) => { - const { destinationResponse, rudderJobMetadata } = responseParams; + const { destinationResponse, rudderJobMetadata, destinationRequest } = responseParams; const successMessage = `[HUBSPOT Response V1 Handler] - Request Processed Successfully`; const failureMessage = 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation'; @@ -37,11 +44,11 @@ const responseHandler = (responseParams) => { if (isHttpStatusSuccess(status)) { // populate different response for each event const results = response?.results; - if (verify(results, rudderJobMetadata)) { - for (const [idx] of rudderJobMetadata.entries()) { + if (verify(results, rudderJobMetadata, destinationRequest?.destinationConfig)) { + for (const metadata of rudderJobMetadata) { const proxyOutputObj = { statusCode: 200, - metadata: rudderJobMetadata[idx], + metadata, error: 'success', }; responseWithIndividualEvents.push(proxyOutputObj); @@ -54,12 +61,15 @@ const responseHandler = (responseParams) => { }; } // return the destiantionResponse as it is when the response is not in expected format - return { + throw new TransformerProxyError( + failureMessage, status, - message: successMessage, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, destinationResponse, - response: destinationResponse, - }; + getAuthErrCategoryFromStCode(status), + ); } // At least one event in the batch is invalid. diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts index 2778ffbff7..dcc14fbd1d 100644 --- a/test/integrations/destinations/hs/dataDelivery/business.ts +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -41,6 +41,9 @@ export const businessData = [ }, }, [generateMetadata(1), generateMetadata(2)], + { + apiVersion: 'newApi', + }, ), }, }, diff --git a/test/integrations/destinations/hs/dataDelivery/other.ts b/test/integrations/destinations/hs/dataDelivery/other.ts index e99541802e..3bb7f3896d 100644 --- a/test/integrations/destinations/hs/dataDelivery/other.ts +++ b/test/integrations/destinations/hs/dataDelivery/other.ts @@ -223,18 +223,29 @@ export const otherData = [ status: 200, body: { output: { - destinationResponse: { - response: { - message: 'unknown response', + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: '{"message":"unknown response"}', + metadata: generateMetadata(1), + statusCode: 200, }, - status: 200, - }, - message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', - response: { - response: { - message: 'unknown response', + { + error: '{"message":"unknown response"}', + metadata: generateMetadata(2), + statusCode: 200, }, - status: 200, + ], + statTags: { + destType: 'HS', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', }, status: 200, }, From 4f1915fcec3ab9edf66dafd476683a91da08841f Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Wed, 12 Jun 2024 18:14:14 +0530 Subject: [PATCH 07/11] chore: address comment --- .../{networkHandler.js => networkHandler.ts} | 59 ++++++---- .../destinations/hs/dataDelivery/business.ts | 111 +++++++++++++++++- test/integrations/destinations/hs/network.ts | 43 +++++++ 3 files changed, 186 insertions(+), 27 deletions(-) rename src/v1/destinations/hs/{networkHandler.js => networkHandler.ts} (68%) diff --git a/src/v1/destinations/hs/networkHandler.js b/src/v1/destinations/hs/networkHandler.ts similarity index 68% rename from src/v1/destinations/hs/networkHandler.js rename to src/v1/destinations/hs/networkHandler.ts index 0e37a64327..84215c3c49 100644 --- a/src/v1/destinations/hs/networkHandler.js +++ b/src/v1/destinations/hs/networkHandler.ts @@ -1,8 +1,7 @@ -/* eslint-disable no-param-reassign */ -/* eslint-disable no-restricted-syntax */ -const { TransformerProxyError } = require('../../../v0/util/errorTypes'); -const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); -const { isHttpStatusSuccess, getAuthErrCategoryFromStCode } = require('../../../v0/util/index'); +import { TransformerProxyError } from '../../../v0/util/errorTypes'; +import { prepareProxyRequest, proxyRequest } from '../../../adapters/network'; +import { isHttpStatusSuccess, getAuthErrCategoryFromStCode } from '../../../v0/util/index'; +import { DeliveryV1Response, DeliveryJobState } from '../../../types/index'; const { processAxiosResponse, @@ -11,54 +10,65 @@ const { const tags = require('../../../v0/util/tags'); const verify = (results, rudderJobMetadata, destinationConfig) => { - if (destinationConfig?.apiVersion === undefined) { - return false; + if (destinationConfig?.apiVersion === 'legacyApi') { + return true; } if (destinationConfig?.apiVersion === 'newApi') { return Array.isArray(results) && results.length === rudderJobMetadata.length; } - return true; + return false; }; -const populateResponseWithDontBatch = (rudderJobMetadata, status, response) => { +const populateResponseWithDontBatch = (rudderJobMetadata, response) => { const errorMessage = JSON.stringify(response); - const responseWithIndividualEvents = []; - for (const metadata of rudderJobMetadata) { - metadata.dontBatch = true; + const responseWithIndividualEvents: DeliveryJobState[] = []; + + rudderJobMetadata.forEach((metadata) => { responseWithIndividualEvents.push({ statusCode: 500, - metadata, + metadata: { ...metadata, dontBatch: true }, error: errorMessage, }); - } + }); return responseWithIndividualEvents; }; + +type Result = { + status?: string; + results?: Array; + startedAt?: Date; + completedAt?: Date; + message?: string; + correlationId?: string; + failureMessages?: Array; +}; + const responseHandler = (responseParams) => { const { destinationResponse, rudderJobMetadata, destinationRequest } = responseParams; const successMessage = `[HUBSPOT Response V1 Handler] - Request Processed Successfully`; const failureMessage = 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation'; - const responseWithIndividualEvents = []; + const responseWithIndividualEvents: DeliveryJobState[] = []; const { response, status } = destinationResponse; if (isHttpStatusSuccess(status)) { // populate different response for each event - const results = response?.results; + const results = (response as Result)?.results; if (verify(results, rudderJobMetadata, destinationRequest?.destinationConfig)) { - for (const metadata of rudderJobMetadata) { - const proxyOutputObj = { + rudderJobMetadata.forEach((metadata) => { + const proxyOutputObj: DeliveryJobState = { statusCode: 200, metadata, error: 'success', }; responseWithIndividualEvents.push(proxyOutputObj); - } + }); + return { status, message: successMessage, - destinationResponse, response: responseWithIndividualEvents, - }; + } as DeliveryV1Response; } // return the destiantionResponse as it is when the response is not in expected format throw new TransformerProxyError( @@ -78,9 +88,8 @@ const responseHandler = (responseParams) => { return { status: 500, message: failureMessage, - destinationResponse, - response: populateResponseWithDontBatch(rudderJobMetadata, status, response), - }; + response: populateResponseWithDontBatch(rudderJobMetadata, response), + } as DeliveryV1Response; } throw new TransformerProxyError( failureMessage, @@ -94,7 +103,7 @@ const responseHandler = (responseParams) => { ); }; -function networkHandler() { +function networkHandler(this: any) { this.prepareProxy = prepareProxyRequest; this.proxy = proxyRequest; this.processAxiosResponse = processAxiosResponse; diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts index dcc14fbd1d..def8621eeb 100644 --- a/test/integrations/destinations/hs/dataDelivery/business.ts +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -13,7 +13,7 @@ const commonStatTags = { export const businessData = [ { name: 'hs', - description: 'successfully creating users from a batch', + description: 'successfully creating users from a batch with new api', feature: 'dataDelivery', module: 'destination', version: 'v1', @@ -114,7 +114,7 @@ export const businessData = [ }, { name: 'hs', - description: 'failed due to duplicate object in a batch', + description: 'failed due to duplicate in a batch', id: 'hs_datadelivery_01', feature: 'dataDelivery', module: 'destination', @@ -283,4 +283,111 @@ export const businessData = [ }, }, }, + { + name: 'hs', + description: 'successfully creating users from a batch with legacy api', + feature: 'dataDelivery', + module: 'destination', + id: 'successWithLegacyApi', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/contacts/v1/contact/batch/', + JSON_ARRAY: { + batch: + '[{"email":"identify111051@test.com","properties":[{"property":"firstname","value":"John1051"},{"property":"lastname","value":"Sparrow1051"}]},{"email":"identify111052@test.com","properties":[{"property":"firstname","value":"John1052"},{"property":"lastname","value":"Sparrow1052"}]},{"email":"identify111053@test.com","properties":[{"property":"firstname","value":"John1053"},{"property":"lastname","value":"Sparrow1053"}]}]', + }, + headers: { + Authorization: 'Bearer validApiKey', + 'Content-Type': 'application/json', + }, + }, + [generateMetadata(1), generateMetadata(2)], + { + apiVersion: 'legacyApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: 'success', + metadata: generateMetadata(1), + statusCode: 200, + }, + { + error: 'success', + metadata: generateMetadata(2), + statusCode: 200, + }, + ], + status: 200, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed to create users from a batch with legacy api', + feature: 'dataDelivery', + module: 'destination', + id: 'failureWithLegacyApi', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/contacts/v1/contact/batch/', + JSON_ARRAY: { + batch: + '[{"email":"identify111051@test.com","properties":[{"property":"firstname","value":"John1051"},{"property":"lastname","value":"Sparrow1051"}]},{"email":"identify111052@test.con","properties":[{"property":"firstname","value":"John1052"},{"property":"lastname","value":"Sparrow1052"}]},{"email":"identify111053@test.com","properties":[{"property":"firstname","value":"John1053"},{"property":"lastname","value":"Sparrow1053"}]}]', + }, + headers: { + Authorization: 'Bearer inValidApiKey', + 'Content-Type': 'application/json', + }, + }, + [generateMetadata(1), generateMetadata(2)], + { + apiVersion: 'legacyApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"status":"error","message":"Errors found processing batch update","correlationId":"a716ef20-79df-44d4-98bd-9136af7bdefc","invalidEmails":["identify111052@test.con"],"failureMessages":[{"index":1,"error":{"status":"error","message":"Email address identify111052@test.con is invalid"}}]}', + metadata: { ...generateMetadata(1), dontBatch: true }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Errors found processing batch update","correlationId":"a716ef20-79df-44d4-98bd-9136af7bdefc","invalidEmails":["identify111052@test.con"],"failureMessages":[{"index":1,"error":{"status":"error","message":"Email address identify111052@test.con is invalid"}}]}', + metadata: { ...generateMetadata(2), dontBatch: true }, + statusCode: 500, + }, + ], + status: 500, + }, + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/hs/network.ts b/test/integrations/destinations/hs/network.ts index a306c20e66..0f68b7d7c1 100644 --- a/test/integrations/destinations/hs/network.ts +++ b/test/integrations/destinations/hs/network.ts @@ -876,4 +876,47 @@ export const networkCallsData = [ status: 200, }, }, + { + httpReq: { + url: 'https://api.hubapi.com/contacts/v1/contact/batch/', + method: 'POST', + headers: { + 'User-Agent': 'RudderLabs', + 'Content-Type': 'application/json', + Authorization: 'Bearer validApiKey', + }, + }, + httpRes: { + status: 200, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/contacts/v1/contact/batch/', + method: 'POST', + headers: { + 'User-Agent': 'RudderLabs', + 'Content-Type': 'application/json', + Authorization: 'Bearer inValidApiKey', + }, + }, + httpRes: { + status: 400, + data: { + status: 'error', + message: 'Errors found processing batch update', + correlationId: 'a716ef20-79df-44d4-98bd-9136af7bdefc', + invalidEmails: ['identify111052@test.con'], + failureMessages: [ + { + index: 1, + error: { + status: 'error', + message: 'Email address identify111052@test.con is invalid', + }, + }, + ], + }, + }, + }, ]; From e87d031276d9368496163fe11b35f9c9bef4b7c1 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Thu, 20 Jun 2024 11:09:08 +0530 Subject: [PATCH 08/11] chore: update failed tests --- .../destinations/hs/dataDelivery/business.ts | 62 ------------------- 1 file changed, 62 deletions(-) diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts index def8621eeb..cd1e871070 100644 --- a/test/integrations/destinations/hs/dataDelivery/business.ts +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -66,47 +66,6 @@ export const businessData = [ statusCode: 200, }, ], - destinationResponse: { - response: { - status: 'COMPLETE', - results: [ - { - id: '12877907025', - properties: { - createdate: '2024-04-16T09:50:16.034Z', - email: 'test1@mail.com', - firstname: 'test1', - hs_is_unworked: 'true', - hs_object_id: '12877907025', - hs_pipeline: 'contacts-lifecycle-pipeline', - lastmodifieddate: '2024-04-23T11:52:03.723Z', - lifecyclestage: 'lead', - }, - createdAt: '2024-04-16T09:50:16.034Z', - updatedAt: '2024-04-23T11:52:03.723Z', - archived: false, - }, - { - id: '12877907024', - properties: { - createdate: '2024-04-16T09:50:16.034Z', - firstname: 'testmail1217', - hs_is_unworked: 'true', - hs_object_id: '12877907024', - hs_pipeline: 'contacts-lifecycle-pipeline', - lastmodifieddate: '2024-04-23T11:52:03.723Z', - lifecyclestage: 'lead', - }, - createdAt: '2024-04-16T09:50:16.034Z', - updatedAt: '2024-04-23T11:52:03.723Z', - archived: false, - }, - ], - startedAt: '2024-04-24T05:11:51.090Z', - completedAt: '2024-04-24T05:11:51.190Z', - }, - status: 200, - }, }, }, }, @@ -152,18 +111,6 @@ export const businessData = [ status: 200, body: { output: { - destinationResponse: { - response: { - category: 'VALIDATION_ERROR', - context: { - ids: ['12877907025'], - }, - correlationId: 'd24ec5cd-8998-4674-a928-59603ae6b0eb', - message: 'Duplicate IDs found in batch input: [12877907025]. IDs must be unique', - status: 'error', - }, - status: 400, - }, message: 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', response: [ @@ -237,15 +184,6 @@ export const businessData = [ status: 200, body: { output: { - destinationResponse: { - response: { - correlationId: '99df04b9-da11-4504-bd97-2c15f58d0943', - message: - 'Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)', - status: 'error', - }, - status: 400, - }, message: 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', response: [ From 722bfbccf758bd74adbb6127924ce0517e21cfc7 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Mon, 15 Jul 2024 12:20:27 +0530 Subject: [PATCH 09/11] fix: update network handler --- src/v1/destinations/hs/networkHandler.ts | 90 ++++++++++++------- .../destinations/hs/dataDelivery/business.ts | 6 +- .../destinations/hs/dataDelivery/other.ts | 17 +--- 3 files changed, 65 insertions(+), 48 deletions(-) diff --git a/src/v1/destinations/hs/networkHandler.ts b/src/v1/destinations/hs/networkHandler.ts index 84215c3c49..79237075d6 100644 --- a/src/v1/destinations/hs/networkHandler.ts +++ b/src/v1/destinations/hs/networkHandler.ts @@ -3,20 +3,31 @@ import { prepareProxyRequest, proxyRequest } from '../../../adapters/network'; import { isHttpStatusSuccess, getAuthErrCategoryFromStCode } from '../../../v0/util/index'; import { DeliveryV1Response, DeliveryJobState } from '../../../types/index'; -const { - processAxiosResponse, - getDynamicErrorType, -} = require('../../../adapters/utils/networkUtils'); +import { processAxiosResponse, getDynamicErrorType } from '../../../adapters/utils/networkUtils'; + const tags = require('../../../v0/util/tags'); -const verify = (results, rudderJobMetadata, destinationConfig) => { +/** + * + * @param results + * @param rudderJobMetadata + * @param destinationConfig + * @returns boolean + */ + +const findFeatureandVersion = (results, rudderJobMetadata, destinationConfig) => { + if (!Array.isArray(rudderJobMetadata)) { + return 'singleEvent'; + } if (destinationConfig?.apiVersion === 'legacyApi') { - return true; + return 'legacyApiWithMultipleEvents'; } if (destinationConfig?.apiVersion === 'newApi') { - return Array.isArray(results) && results.length === rudderJobMetadata.length; + if (Array.isArray(results) && results.length === rudderJobMetadata.length) + return 'newApiWithMultipleEvents'; + return 'newApiWithMultipleEventsWrongLength'; } - return false; + return 'unableToFindVersionWithMultipleEvents'; }; const populateResponseWithDontBatch = (rudderJobMetadata, response) => { @@ -54,36 +65,51 @@ const responseHandler = (responseParams) => { if (isHttpStatusSuccess(status)) { // populate different response for each event const results = (response as Result)?.results; - if (verify(results, rudderJobMetadata, destinationRequest?.destinationConfig)) { - rudderJobMetadata.forEach((metadata) => { - const proxyOutputObj: DeliveryJobState = { - statusCode: 200, - metadata, - error: 'success', + let proxyOutputObj: DeliveryJobState; + const featureAndVersion = findFeatureandVersion( + results, + rudderJobMetadata, + destinationRequest?.destinationConfig, + ); + switch (featureAndVersion) { + case 'singleEvent': + proxyOutputObj = { + statusCode: status, + metadata: rudderJobMetadata, + error: response, }; responseWithIndividualEvents.push(proxyOutputObj); - }); - - return { - status, - message: successMessage, - response: responseWithIndividualEvents, - } as DeliveryV1Response; + break; + case 'newApiWithMultipleEvents': + rudderJobMetadata.forEach((metadata, index) => { + proxyOutputObj = { + statusCode: 200, + metadata, + error: JSON.stringify(results?.[index]), + }; + responseWithIndividualEvents.push(proxyOutputObj); + }); + break; + default: + rudderJobMetadata.forEach((metadata) => { + proxyOutputObj = { + statusCode: 200, + metadata, + error: 'success', + }; + responseWithIndividualEvents.push(proxyOutputObj); + }); + break; } - // return the destiantionResponse as it is when the response is not in expected format - throw new TransformerProxyError( - failureMessage, + return { status, - { - [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), - }, - destinationResponse, - getAuthErrCategoryFromStCode(status), - ); + message: successMessage, + response: responseWithIndividualEvents, + } as DeliveryV1Response; } // At least one event in the batch is invalid. - if (status === 400 && rudderJobMetadata.length > 1) { + if (status === 400 && Array.isArray(rudderJobMetadata) && rudderJobMetadata.length > 1) { // sending back 500 for retry only when events came in a batch return { status: 500, @@ -99,7 +125,7 @@ const responseHandler = (responseParams) => { }, destinationResponse, getAuthErrCategoryFromStCode(status), - responseWithIndividualEvents, + response, ); }; diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts index cd1e871070..f9b3de51f2 100644 --- a/test/integrations/destinations/hs/dataDelivery/business.ts +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -56,12 +56,14 @@ export const businessData = [ message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', response: [ { - error: 'success', + error: + '{"id":"12877907025","properties":{"createdate":"2024-04-16T09:50:16.034Z","email":"test1@mail.com","firstname":"test1","hs_is_unworked":"true","hs_object_id":"12877907025","hs_pipeline":"contacts-lifecycle-pipeline","lastmodifieddate":"2024-04-23T11:52:03.723Z","lifecyclestage":"lead"},"createdAt":"2024-04-16T09:50:16.034Z","updatedAt":"2024-04-23T11:52:03.723Z","archived":false}', metadata: generateMetadata(1), statusCode: 200, }, { - error: 'success', + error: + '{"id":"12877907024","properties":{"createdate":"2024-04-16T09:50:16.034Z","firstname":"testmail1217","hs_is_unworked":"true","hs_object_id":"12877907024","hs_pipeline":"contacts-lifecycle-pipeline","lastmodifieddate":"2024-04-23T11:52:03.723Z","lifecyclestage":"lead"},"createdAt":"2024-04-16T09:50:16.034Z","updatedAt":"2024-04-23T11:52:03.723Z","archived":false}', metadata: generateMetadata(2), statusCode: 200, }, diff --git a/test/integrations/destinations/hs/dataDelivery/other.ts b/test/integrations/destinations/hs/dataDelivery/other.ts index 3bb7f3896d..202b665a51 100644 --- a/test/integrations/destinations/hs/dataDelivery/other.ts +++ b/test/integrations/destinations/hs/dataDelivery/other.ts @@ -223,30 +223,19 @@ export const otherData = [ status: 200, body: { output: { - message: - 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', response: [ { - error: '{"message":"unknown response"}', + error: 'success', metadata: generateMetadata(1), statusCode: 200, }, { - error: '{"message":"unknown response"}', + error: 'success', metadata: generateMetadata(2), statusCode: 200, }, ], - statTags: { - destType: 'HS', - destinationId: 'default-destinationId', - errorCategory: 'network', - errorType: 'aborted', - feature: 'dataDelivery', - implementation: 'native', - module: 'destination', - workspaceId: 'default-workspaceId', - }, status: 200, }, }, From 2fe5fbd53370ef47e980c1baee92357913900cb3 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Wed, 17 Jul 2024 11:10:11 +0530 Subject: [PATCH 10/11] feat: add verification of single event --- src/v1/destinations/hs/networkHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v1/destinations/hs/networkHandler.ts b/src/v1/destinations/hs/networkHandler.ts index 79237075d6..108b139ee6 100644 --- a/src/v1/destinations/hs/networkHandler.ts +++ b/src/v1/destinations/hs/networkHandler.ts @@ -16,7 +16,7 @@ const tags = require('../../../v0/util/tags'); */ const findFeatureandVersion = (results, rudderJobMetadata, destinationConfig) => { - if (!Array.isArray(rudderJobMetadata)) { + if (Array.isArray(rudderJobMetadata) && rudderJobMetadata.length === 1) { return 'singleEvent'; } if (destinationConfig?.apiVersion === 'legacyApi') { From 653e25c0419d197b0d88ebc1824fcbdba3692de4 Mon Sep 17 00:00:00 2001 From: ItsSudip Date: Wed, 31 Jul 2024 10:33:19 +0530 Subject: [PATCH 11/11] chore: add test cases --- src/v0/destinations/hs/transform.js | 1 + src/v1/destinations/hs/networkHandler.ts | 26 +- .../destinations/hs/dataDelivery/business.ts | 336 ++++++++++++++++-- test/integrations/destinations/hs/network.ts | 149 ++++++++ 4 files changed, 465 insertions(+), 47 deletions(-) diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js index 9ab3b8c790..1d1a81b6b0 100644 --- a/src/v0/destinations/hs/transform.js +++ b/src/v0/destinations/hs/transform.js @@ -192,6 +192,7 @@ const processRouterDest = async (inputs, reqMetadata) => { batchedResponseList.push(...response.batchedResponseList); dontBatchEvents.push(...response.dontBatchEvents); }); + console.log(JSON.stringify([...batchedResponseList, ...errorRespList, ...dontBatchEvents])); return [...batchedResponseList, ...errorRespList, ...dontBatchEvents]; }; diff --git a/src/v1/destinations/hs/networkHandler.ts b/src/v1/destinations/hs/networkHandler.ts index 108b139ee6..8293a7240c 100644 --- a/src/v1/destinations/hs/networkHandler.ts +++ b/src/v1/destinations/hs/networkHandler.ts @@ -15,7 +15,8 @@ const tags = require('../../../v0/util/tags'); * @returns boolean */ -const findFeatureandVersion = (results, rudderJobMetadata, destinationConfig) => { +const findFeatureandVersion = (response, rudderJobMetadata, destinationConfig) => { + const { results, errors } = response; if (Array.isArray(rudderJobMetadata) && rudderJobMetadata.length === 1) { return 'singleEvent'; } @@ -25,7 +26,13 @@ const findFeatureandVersion = (results, rudderJobMetadata, destinationConfig) => if (destinationConfig?.apiVersion === 'newApi') { if (Array.isArray(results) && results.length === rudderJobMetadata.length) return 'newApiWithMultipleEvents'; - return 'newApiWithMultipleEventsWrongLength'; + + if ( + Array.isArray(results) && + results.length !== rudderJobMetadata.length && + results.length + errors.length === rudderJobMetadata.length + ) + return 'newApiWithMultipleEventsAndErrors'; } return 'unableToFindVersionWithMultipleEvents'; }; @@ -44,9 +51,10 @@ const populateResponseWithDontBatch = (rudderJobMetadata, response) => { return responseWithIndividualEvents; }; -type Result = { +type Response = { status?: string; results?: Array; + errors?: Array; startedAt?: Date; completedAt?: Date; message?: string; @@ -64,10 +72,10 @@ const responseHandler = (responseParams) => { if (isHttpStatusSuccess(status)) { // populate different response for each event - const results = (response as Result)?.results; + const destResponse = response as Response; let proxyOutputObj: DeliveryJobState; const featureAndVersion = findFeatureandVersion( - results, + destResponse, rudderJobMetadata, destinationRequest?.destinationConfig, ); @@ -75,17 +83,17 @@ const responseHandler = (responseParams) => { case 'singleEvent': proxyOutputObj = { statusCode: status, - metadata: rudderJobMetadata, - error: response, + metadata: rudderJobMetadata[0], + error: JSON.stringify(destResponse), }; responseWithIndividualEvents.push(proxyOutputObj); break; case 'newApiWithMultipleEvents': - rudderJobMetadata.forEach((metadata, index) => { + rudderJobMetadata.forEach((metadata: any, index: string | number) => { proxyOutputObj = { statusCode: 200, metadata, - error: JSON.stringify(results?.[index]), + error: JSON.stringify(destResponse.results?.[index]), }; responseWithIndividualEvents.push(proxyOutputObj); }); diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts index f9b3de51f2..2239abfb95 100644 --- a/test/integrations/destinations/hs/dataDelivery/business.ts +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -11,12 +11,234 @@ const commonStatTags = { workspaceId: 'default-workspaceId', }; export const businessData = [ + { + name: 'hs', + description: 'successfully creating users from a batch with legacy api', + feature: 'dataDelivery', + module: 'destination', + id: 'successWithLegacyApi', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/contacts/v1/contact/batch/', + JSON_ARRAY: { + batch: + '[{"email":"identify111051@test.com","properties":[{"property":"firstname","value":"John1051"},{"property":"lastname","value":"Sparrow1051"}]},{"email":"identify111052@test.com","properties":[{"property":"firstname","value":"John1052"},{"property":"lastname","value":"Sparrow1052"}]},{"email":"identify111053@test.com","properties":[{"property":"firstname","value":"John1053"},{"property":"lastname","value":"Sparrow1053"}]}]', + }, + headers: { + Authorization: 'Bearer validApiKey', + 'Content-Type': 'application/json', + }, + }, + [generateMetadata(1), generateMetadata(2)], + { + apiVersion: 'legacyApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: 'success', + metadata: generateMetadata(1), + statusCode: 200, + }, + { + error: 'success', + metadata: generateMetadata(2), + statusCode: 200, + }, + ], + status: 200, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed to create users from a batch with legacy api', + feature: 'dataDelivery', + module: 'destination', + id: 'failureWithLegacyApi', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/contacts/v1/contact/batch/', + JSON_ARRAY: { + batch: + '[{"email":"identify111051@test.com","properties":[{"property":"firstname","value":"John1051"},{"property":"lastname","value":"Sparrow1051"}]},{"email":"identify111052@test.con","properties":[{"property":"firstname","value":"John1052"},{"property":"lastname","value":"Sparrow1052"}]},{"email":"identify111053@test.com","properties":[{"property":"firstname","value":"John1053"},{"property":"lastname","value":"Sparrow1053"}]}]', + }, + headers: { + Authorization: 'Bearer inValidApiKey', + 'Content-Type': 'application/json', + }, + }, + [generateMetadata(1), generateMetadata(2)], + { + apiVersion: 'legacyApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"status":"error","message":"Errors found processing batch update","correlationId":"a716ef20-79df-44d4-98bd-9136af7bdefc","invalidEmails":["identify111052@test.con"],"failureMessages":[{"index":1,"error":{"status":"error","message":"Email address identify111052@test.con is invalid"}}]}', + metadata: { ...generateMetadata(1), dontBatch: true }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Errors found processing batch update","correlationId":"a716ef20-79df-44d4-98bd-9136af7bdefc","invalidEmails":["identify111052@test.con"],"failureMessages":[{"index":1,"error":{"status":"error","message":"Email address identify111052@test.con is invalid"}}]}', + metadata: { ...generateMetadata(2), dontBatch: true }, + statusCode: 500, + }, + ], + status: 500, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'successfully deliver events with legacy api', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + id: 'successEventsWithLegacyApi', + input: { + request: { + body: generateProxyV1Payload( + { + headers: { + 'Content-Type': 'application/json', + }, + method: 'GET', + params: { + _a: 'dummy-hubId', + _n: 'test track event HS', + _m: 4.99, + email: 'testhubspot2@email.com', + firstname: 'Test Hubspot', + }, + endpoint: 'https://track.hubspot.com/v1/event', + }, + [generateMetadata(1)], + { + apiVersion: 'legacyApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: '{}', + metadata: generateMetadata(1), + statusCode: 200, + }, + ], + status: 200, + }, + }, + }, + }, + }, { name: 'hs', description: 'successfully creating users from a batch with new api', feature: 'dataDelivery', module: 'destination', version: 'v1', + id: 'successCreatingBatchOfUsersWithNewApi', + input: { + request: { + body: generateProxyV1Payload( + { + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer validAccessToken', + }, + method: 'POST', + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/create', + JSON: { + inputs: [ + { + properties: { + email: 'testuser31848@testmail.com', + }, + }, + { + properties: { + email: 'testuser31847@testmail.com', + }, + }, + ], + }, + }, + [generateMetadata(1), generateMetadata(2)], + { + apiVersion: 'newApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: + '{"id":"44188066992","properties":{"createdate":"2024-07-31T03:21:03.176Z","email":"testuser31848@testmail.com","hs_all_contact_vids":"44188066992","hs_email_domain":"testmail.com","hs_is_contact":"true","hs_is_unworked":"true","hs_lifecyclestage_lead_date":"2024-07-31T03:21:03.176Z","hs_membership_has_accessed_private_content":"0","hs_object_id":"44188066992","hs_object_source":"INTEGRATION","hs_object_source_id":"3209723","hs_object_source_label":"INTEGRATION","hs_pipeline":"contacts-lifecycle-pipeline","hs_registered_member":"0","lastmodifieddate":"2024-07-31T03:21:03.176Z","lifecyclestage":"lead"},"createdAt":"2024-07-31T03:21:03.176Z","updatedAt":"2024-07-31T03:21:03.176Z","archived":false}', + metadata: generateMetadata(1), + statusCode: 200, + }, + { + error: + '{"id":"44188066993","properties":{"createdate":"2024-07-31T03:21:03.176Z","email":"testuser31847@testmail.com","hs_all_contact_vids":"44188066993","hs_email_domain":"testmail.com","hs_is_contact":"true","hs_is_unworked":"true","hs_lifecyclestage_lead_date":"2024-07-31T03:21:03.176Z","hs_membership_has_accessed_private_content":"0","hs_object_id":"44188066993","hs_object_source":"INTEGRATION","hs_object_source_id":"3209723","hs_object_source_label":"INTEGRATION","hs_pipeline":"contacts-lifecycle-pipeline","hs_registered_member":"0","lastmodifieddate":"2024-07-31T03:21:03.176Z","lifecyclestage":"lead"},"createdAt":"2024-07-31T03:21:03.176Z","updatedAt":"2024-07-31T03:21:03.176Z","archived":false}', + metadata: generateMetadata(2), + statusCode: 200, + }, + ], + status: 200, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'successfully updating users from a batch with new api', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', input: { request: { body: generateProxyV1Payload( @@ -225,29 +447,30 @@ export const businessData = [ }, { name: 'hs', - description: 'successfully creating users from a batch with legacy api', + description: 'succeed to send a custom event', feature: 'dataDelivery', module: 'destination', - id: 'successWithLegacyApi', + id: 'succeedEventWithNewApi', version: 'v1', input: { request: { body: generateProxyV1Payload( { - endpoint: 'https://api.hubapi.com/contacts/v1/contact/batch/', - JSON_ARRAY: { - batch: - '[{"email":"identify111051@test.com","properties":[{"property":"firstname","value":"John1051"},{"property":"lastname","value":"Sparrow1051"}]},{"email":"identify111052@test.com","properties":[{"property":"firstname","value":"John1052"},{"property":"lastname","value":"Sparrow1052"}]},{"email":"identify111053@test.com","properties":[{"property":"firstname","value":"John1053"},{"property":"lastname","value":"Sparrow1053"}]}]', - }, + endpoint: 'https://api.hubapi.com/events/v3/send', + method: 'POST', headers: { - Authorization: 'Bearer validApiKey', 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + JSON: { + email: 'osvaldocostaferreira98@gmail.com', + eventName: 'pe22315509_rs_hub_test', + properties: { + value: 'name1', + }, }, }, - [generateMetadata(1), generateMetadata(2)], - { - apiVersion: 'legacyApi', - }, + [generateMetadata(1)], ), }, }, @@ -259,15 +482,10 @@ export const businessData = [ message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', response: [ { - error: 'success', + error: '{}', metadata: generateMetadata(1), statusCode: 200, }, - { - error: 'success', - metadata: generateMetadata(2), - statusCode: 200, - }, ], status: 200, }, @@ -277,54 +495,96 @@ export const businessData = [ }, { name: 'hs', - description: 'failed to create users from a batch with legacy api', + description: 'succeed to send a custom event', feature: 'dataDelivery', module: 'destination', - id: 'failureWithLegacyApi', + id: 'failedEventWithNewApi', version: 'v1', input: { request: { body: generateProxyV1Payload( { - endpoint: 'https://api.hubapi.com/contacts/v1/contact/batch/', - JSON_ARRAY: { - batch: - '[{"email":"identify111051@test.com","properties":[{"property":"firstname","value":"John1051"},{"property":"lastname","value":"Sparrow1051"}]},{"email":"identify111052@test.con","properties":[{"property":"firstname","value":"John1052"},{"property":"lastname","value":"Sparrow1052"}]},{"email":"identify111053@test.com","properties":[{"property":"firstname","value":"John1053"},{"property":"lastname","value":"Sparrow1053"}]}]', - }, + endpoint: 'https://api.hubapi.com/events/v3/send', + method: 'POST', headers: { - Authorization: 'Bearer inValidApiKey', 'Content-Type': 'application/json', + Authorization: 'Bearer invalid-dummy-access-token', + }, + JSON: { + email: 'osvaldocostaferreira98@gmail.com', + eventName: 'pe22315509_rs_hub_test', + properties: { + value: 'name1', + }, }, }, - [generateMetadata(1), generateMetadata(2)], - { - apiVersion: 'legacyApi', - }, + [generateMetadata(1)], ), }, }, output: { response: { - status: 200, + status: 401, body: { output: { + authErrorCategory: 'REFRESH_TOKEN', message: 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', response: [ { error: - '{"status":"error","message":"Errors found processing batch update","correlationId":"a716ef20-79df-44d4-98bd-9136af7bdefc","invalidEmails":["identify111052@test.con"],"failureMessages":[{"index":1,"error":{"status":"error","message":"Email address identify111052@test.con is invalid"}}]}', - metadata: { ...generateMetadata(1), dontBatch: true }, - statusCode: 500, + '{"status":"error","message":"Authentication credentials not found. This API supports OAuth 2.0 authentication and you can find more details at https://developers.hubspot.com/docs/methods/auth/oauth-overview","correlationId":"501651f6-bb90-40f1-b0db-349f62916993","category":"INVALID_AUTHENTICATION"}', + metadata: generateMetadata(1), + statusCode: 401, }, + ], + statTags: { ...commonStatTags, errorType: 'aborted' }, + status: 401, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'succeed to send an event with association', + feature: 'dataDelivery', + module: 'destination', + id: 'succeedAssociationWithNewApi', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/associations/companies/contacts/batch/create', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + JSON: { + inputs: [{ to: { id: 1 }, from: { id: 9405415215 }, type: 'contact_to_company' }], + }, + }, + [generateMetadata(1)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ { error: - '{"status":"error","message":"Errors found processing batch update","correlationId":"a716ef20-79df-44d4-98bd-9136af7bdefc","invalidEmails":["identify111052@test.con"],"failureMessages":[{"index":1,"error":{"status":"error","message":"Email address identify111052@test.con is invalid"}}]}', - metadata: { ...generateMetadata(2), dontBatch: true }, - statusCode: 500, + '{"completedAt":"2024-07-31T04:46:34.391Z","requestedAt":"2024-07-31T04:46:34.391Z","startedAt":"2024-07-31T04:46:34.391Z","results":[{"from":{"id":"9405415215"},"to":{"id":"1"},"type":"contact_to_company"}],"status":"PENDING"}', + metadata: generateMetadata(1), + statusCode: 201, }, ], - status: 500, + status: 201, }, }, }, diff --git a/test/integrations/destinations/hs/network.ts b/test/integrations/destinations/hs/network.ts index 0f68b7d7c1..9d8658ff6b 100644 --- a/test/integrations/destinations/hs/network.ts +++ b/test/integrations/destinations/hs/network.ts @@ -919,4 +919,153 @@ export const networkCallsData = [ }, }, }, + { + httpReq: { + url: 'https://track.hubspot.com/v1/event', + method: 'GET', + params: { + _a: 'dummy-hubId', + _n: 'test track event HS', + _m: 4.99, + email: 'testhubspot2@email.com', + firstname: 'Test Hubspot', + }, + }, + httpRes: { + status: 200, + data: {}, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/create', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer validAccessToken', + }, + }, + httpRes: { + status: 200, + data: { + status: 'COMPLETE', + results: [ + { + id: '44188066992', + properties: { + createdate: '2024-07-31T03:21:03.176Z', + email: 'testuser31848@testmail.com', + hs_all_contact_vids: '44188066992', + hs_email_domain: 'testmail.com', + hs_is_contact: 'true', + hs_is_unworked: 'true', + hs_lifecyclestage_lead_date: '2024-07-31T03:21:03.176Z', + hs_membership_has_accessed_private_content: '0', + hs_object_id: '44188066992', + hs_object_source: 'INTEGRATION', + hs_object_source_id: '3209723', + hs_object_source_label: 'INTEGRATION', + hs_pipeline: 'contacts-lifecycle-pipeline', + hs_registered_member: '0', + lastmodifieddate: '2024-07-31T03:21:03.176Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-07-31T03:21:03.176Z', + updatedAt: '2024-07-31T03:21:03.176Z', + archived: false, + }, + { + id: '44188066993', + properties: { + createdate: '2024-07-31T03:21:03.176Z', + email: 'testuser31847@testmail.com', + hs_all_contact_vids: '44188066993', + hs_email_domain: 'testmail.com', + hs_is_contact: 'true', + hs_is_unworked: 'true', + hs_lifecyclestage_lead_date: '2024-07-31T03:21:03.176Z', + hs_membership_has_accessed_private_content: '0', + hs_object_id: '44188066993', + hs_object_source: 'INTEGRATION', + hs_object_source_id: '3209723', + hs_object_source_label: 'INTEGRATION', + hs_pipeline: 'contacts-lifecycle-pipeline', + hs_registered_member: '0', + lastmodifieddate: '2024-07-31T03:21:03.176Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-07-31T03:21:03.176Z', + updatedAt: '2024-07-31T03:21:03.176Z', + archived: false, + }, + ], + startedAt: '2024-07-31T03:21:03.133Z', + completedAt: '2024-07-31T03:21:03.412Z', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/events/v3/send', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + }, + httpRes: { + status: 200, + data: {}, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/events/v3/send', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer invalid-dummy-access-token', + }, + }, + httpRes: { + status: 401, + data: { + status: 'error', + message: + 'Authentication credentials not found. This API supports OAuth 2.0 authentication and you can find more details at https://developers.hubspot.com/docs/methods/auth/oauth-overview', + correlationId: '501651f6-bb90-40f1-b0db-349f62916993', + category: 'INVALID_AUTHENTICATION', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/associations/companies/contacts/batch/create', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + }, + httpRes: { + status: 201, + data: { + completedAt: '2024-07-31T04:46:34.391Z', + requestedAt: '2024-07-31T04:46:34.391Z', + startedAt: '2024-07-31T04:46:34.391Z', + results: [ + { + from: { + id: '9405415215', + }, + to: { + id: '1', + }, + type: 'contact_to_company', + }, + ], + status: 'PENDING', + }, + }, + }, ];