diff --git a/src/v0/destinations/hs/HSTransform-v2.js b/src/v0/destinations/hs/HSTransform-v2.js index 2acdd82152..3699e1c789 100644 --- a/src/v0/destinations/hs/HSTransform-v2.js +++ b/src/v0/destinations/hs/HSTransform-v2.js @@ -12,7 +12,6 @@ const { defaultPatchRequestConfig, getFieldValueFromMessage, getSuccessRespEvents, - addExternalIdToTraits, defaultBatchRequestConfig, removeUndefinedAndNullValues, getDestinationExternalID, @@ -42,6 +41,7 @@ const { getEventAndPropertiesFromConfig, getHsSearchId, populateTraits, + addExternalIdToHSTraits, } = require('./util'); const { JSON_MIME_TYPE } = require('../../util/constant'); @@ -110,7 +110,7 @@ const processIdentify = async (message, destination, propertyMap) => { GENERIC_TRUE_VALUES.includes(mappedToDestination.toString()) && operation ) { - addExternalIdToTraits(message); + addExternalIdToHSTraits(message); if (!objectType) { throw new InstrumentationError('objectType not found'); } diff --git a/src/v0/destinations/hs/config.js b/src/v0/destinations/hs/config.js index fb9790f0e5..67ad3b5bed 100644 --- a/src/v0/destinations/hs/config.js +++ b/src/v0/destinations/hs/config.js @@ -84,6 +84,9 @@ const RETL_SOURCE = 'rETL'; const mappingConfig = getMappingConfig(ConfigCategory, __dirname); const hsCommonConfigJson = mappingConfig[ConfigCategory.COMMON.name]; +const primaryToSecondaryFields = { + email: 'hs_additional_emails', +}; module.exports = { BASE_ENDPOINT, CONTACT_PROPERTY_MAP_ENDPOINT, @@ -112,5 +115,6 @@ module.exports = { RETL_SOURCE, RETL_CREATE_ASSOCIATION_OPERATION, MAX_CONTACTS_PER_REQUEST, + primaryToSecondaryFields, DESTINATION: 'HS', }; diff --git a/src/v0/destinations/hs/util.js b/src/v0/destinations/hs/util.js index 838da08b3b..ffb2df5237 100644 --- a/src/v0/destinations/hs/util.js +++ b/src/v0/destinations/hs/util.js @@ -1,5 +1,6 @@ /* eslint-disable no-await-in-loop */ const lodash = require('lodash'); +const set = require('set-value'); const get = require('get-value'); const { NetworkInstrumentationError, @@ -28,6 +29,7 @@ const { IDENTIFY_CRM_SEARCH_ALL_OBJECTS, SEARCH_LIMIT_VALUE, hsCommonConfigJson, + primaryToSecondaryFields, DESTINATION, MAX_CONTACTS_PER_REQUEST, } = require('./config'); @@ -576,16 +578,30 @@ const performHubSpotSearch = async ( checkAfter = after; // assigning to the new value if no after we assign it to 0 and no more calls will take place const results = processedResponse.response?.results; + const extraProp = primaryToSecondaryFields[identifierType]; if (results) { searchResults.push( - ...results.map((result) => ({ - id: result.id, - property: result.properties[identifierType], - })), + ...results.map((result) => { + const contact = { + id: result.id, + property: result.properties[identifierType], + }; + // Following maps the extra property to the contact object which + // help us to know if the contact was found using secondary property + if (extraProp) { + contact[extraProp] = result.properties?.[extraProp]; + } + return contact; + }), ); } } - + /* + searchResults = { + id: 'existing_contact_id', + property: 'existing_contact_email', // when email is identifier + hs_additional_emails: ['secondary_email'] // when email is identifier + } */ return searchResults; }; @@ -612,7 +628,25 @@ const getRequestData = (identifierType, chunk) => { limit: SEARCH_LIMIT_VALUE, after: 0, }; - + /* In case of email as identifier we add a filter for hs_additional_emails field + * and append hs_additional_emails to properties list + * We are doing this because there might be emails exisitng as hs_additional_emails for some conatct but + * will not come up in search API until we search with hs_additional_emails as well. + * Not doing this resulted in erro 409 Duplicate records found + */ + const secondaryProp = primaryToSecondaryFields[identifierType]; + if (secondaryProp) { + requestData.filterGroups.push({ + filters: [ + { + propertyName: secondaryProp, + values: chunk, + operator: 'IN', + }, + ], + }); + requestData.properties.push(secondaryProp); + } return requestData; }; @@ -623,7 +657,7 @@ const getRequestData = (identifierType, chunk) => { */ const getExistingContactsData = async (inputs, destination) => { const { Config } = destination; - const updateHubspotIds = []; + const hsIdsToBeUpdated = []; const firstMessage = inputs[0].message; if (!firstMessage) { @@ -651,13 +685,19 @@ const getExistingContactsData = async (inputs, destination) => { destination, ); if (searchResults.length > 0) { - updateHubspotIds.push(...searchResults); + hsIdsToBeUpdated.push(...searchResults); } } - return updateHubspotIds; + return hsIdsToBeUpdated; }; - -const setHsSearchId = (input, id) => { +/** + * This functions sets HsSearchId in the externalId array + * @param {*} input -> Input message + * @param {*} id -> Id to be added + * @param {*} useSecondaryProp -> Let us know if that id was found using secondary property and not primnary + * @returns + */ +const setHsSearchId = (input, id, useSecondaryProp = false) => { const { message } = input; const resultExternalId = []; const externalIdArray = message.context?.externalId; @@ -668,6 +708,11 @@ const setHsSearchId = (input, id) => { if (type.includes(DESTINATION)) { extIdObjParam.hsSearchId = id; } + if (useSecondaryProp) { + // we are using it so that when final payload is made + // then primary key shouldn't be overidden + extIdObjParam.useSecondaryObject = useSecondaryProp; + } resultExternalId.push(extIdObjParam); }); } @@ -680,20 +725,24 @@ const setHsSearchId = (input, id) => { * We do search for all the objects before router transform and assign the type (create/update) * accordingly to context.hubspotOperation * + * For email as primary key we use `hs_additional_emails` as well property to search existing contacts * */ const splitEventsForCreateUpdate = async (inputs, destination) => { // get all the id and properties of already existing objects needed for update. - const updateHubspotIds = await getExistingContactsData(inputs, destination); + const hsIdsToBeUpdated = await getExistingContactsData(inputs, destination); const resultInput = inputs.map((input) => { const { message } = input; const inputParam = input; - const { destinationExternalId } = getDestinationExternalIDInfoForRetl(message, DESTINATION); + const { destinationExternalId, identifierType } = getDestinationExternalIDInfoForRetl( + message, + DESTINATION, + ); - const filteredInfo = updateHubspotIds.filter( + const filteredInfo = hsIdsToBeUpdated.filter( (update) => - update.property.toString().toLowerCase() === destinationExternalId.toString().toLowerCase(), + update.property.toString().toLowerCase() === destinationExternalId.toString().toLowerCase(), // second condition is for secondary property for identifier type ); if (filteredInfo.length > 0) { @@ -701,6 +750,26 @@ const splitEventsForCreateUpdate = async (inputs, destination) => { inputParam.message.context.hubspotOperation = 'updateObject'; return inputParam; } + const secondaryProp = primaryToSecondaryFields[identifierType]; + if (secondaryProp) { + // second condition is for secondary property for identifier type + const filteredInfoForSecondaryProp = hsIdsToBeUpdated.filter((update) => + update[secondaryProp] + ?.toString() + .toLowerCase() + .includes(destinationExternalId.toString().toLowerCase()), + ); + if (filteredInfoForSecondaryProp.length > 0) { + inputParam.message.context.externalId = setHsSearchId( + input, + filteredInfoForSecondaryProp[0].id, + true, + ); + inputParam.message.context.hubspotOperation = 'updateObject'; + return inputParam; + } + } + // if not found in the existing contacts, then it's a new contact inputParam.message.context.hubspotOperation = 'createObject'; return inputParam; }); @@ -748,8 +817,22 @@ const populateTraits = async (propertyMap, traits, destination) => { return populatedTraits; }; +const addExternalIdToHSTraits = (message) => { + const externalIdObj = message.context?.externalId?.[0]; + if (externalIdObj.useSecondaryObject) { + /* this condition help us to NOT override the primary key value with the secondary key value + example: + for `email` as primary key and `hs_additonal_emails` as secondary key we don't want to override `email` with `hs_additional_emails`. + neither we want to map anything for `hs_additional_emails` as this property can not be set + */ + return; + } + set(getFieldValueFromMessage(message, 'traits'), externalIdObj.identifierType, externalIdObj.id); +}; + module.exports = { validateDestinationConfig, + addExternalIdToHSTraits, formatKey, fetchFinalSetOfTraits, getProperties, diff --git a/src/v0/destinations/hs/util.test.js b/src/v0/destinations/hs/util.test.js index 30e89d3aee..ea2e10dc3d 100644 --- a/src/v0/destinations/hs/util.test.js +++ b/src/v0/destinations/hs/util.test.js @@ -4,6 +4,7 @@ const { validatePayloadDataTypes, getObjectAndIdentifierType, } = require('./util'); +const { primaryToSecondaryFields } = require('./config'); const propertyMap = { firstName: 'string', @@ -205,7 +206,7 @@ describe('extractUniqueValues utility test cases', () => { describe('getRequestDataAndRequestOptions utility test cases', () => { it('Should return an object with requestData and requestOptions', () => { const identifierType = 'email'; - const chunk = 'test1@gmail.com'; + const chunk = ['test1@gmail.com']; const accessToken = 'dummyAccessToken'; const expectedRequestData = { @@ -219,8 +220,17 @@ describe('getRequestDataAndRequestOptions utility test cases', () => { }, ], }, + { + filters: [ + { + propertyName: primaryToSecondaryFields[identifierType], + values: chunk, + operator: 'IN', + }, + ], + }, ], - properties: [identifierType], + properties: [identifierType, primaryToSecondaryFields[identifierType]], limit: 100, after: 0, }; diff --git a/test/integrations/destinations/hs/network.ts b/test/integrations/destinations/hs/network.ts index e29cc27562..3d3b8fd83f 100644 --- a/test/integrations/destinations/hs/network.ts +++ b/test/integrations/destinations/hs/network.ts @@ -460,6 +460,37 @@ export const networkCallsData = [ status: 200, }, }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/search', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token-hs-additonal-email', + }, + }, + httpRes: { + data: { + total: 1, + results: [ + { + id: '103689', + properties: { + createdate: '2022-07-15T15:25:08.975Z', + email: 'primary@email.com', + hs_object_id: '103604', + hs_additional_emails: 'abc@extraemail.com;secondary@email.com', + lastmodifieddate: '2022-07-15T15:26:49.590Z', + }, + createdAt: '2022-07-15T15:25:08.975Z', + updatedAt: '2022-07-15T15:26:49.590Z', + archived: false, + }, + ], + }, + status: 200, + }, + }, { httpReq: { url: 'https://api.hubapi.com/crm/v3/objects/contacts/search', diff --git a/test/integrations/destinations/hs/router/data.ts b/test/integrations/destinations/hs/router/data.ts index 3a30232f9f..e1c3e04356 100644 --- a/test/integrations/destinations/hs/router/data.ts +++ b/test/integrations/destinations/hs/router/data.ts @@ -1688,4 +1688,145 @@ export const data = [ }, }, }, + { + name: 'hs', + description: 'getting duplicate records for secondary property', + feature: 'router', + module: 'destination', + version: 'v0', + scenario: 'buisness', + successCriteria: + 'should return 200 status code with contact needs to be updated and no email property', + input: { + request: { + body: { + input: [ + { + message: { + type: 'identify', + sentAt: '2024-03-19T18:46:36.348Z', + traits: { + lastname: 'Peñarete', + firstname: 'Karen', + }, + userId: 'secondary@email.com', + channel: 'sources', + context: { + externalId: [ + { + id: 'secondary@email.com', + type: 'HS-contacts', + identifierType: 'email', + }, + ], + mappedToDestination: 'true', + }, + originalTimestamp: '2024-03-19T18:46:36.348Z', + }, + metadata: { jobId: 3, userId: 'u1' }, + destination: { + Config: { + authorizationType: 'newPrivateAppApi', + accessToken: 'dummy-access-token-hs-additonal-email', + hubID: 'dummy-hubId', + apiKey: 'dummy-apikey', + apiVersion: 'newApi', + lookupField: 'email', + hubspotEvents: [], + }, + secretConfig: {}, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + name: 'Hubspot', + enabled: true, + workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', + deleted: false, + createdAt: '2020-12-30T08:39:32.005Z', + updatedAt: '2021-02-03T16:22:31.374Z', + destinationDefinition: { + id: '1aIXqM806xAVm92nx07YwKbRrO9', + name: 'HS', + displayName: 'Hubspot', + createdAt: '2020-04-09T09:24:31.794Z', + updatedAt: '2021-01-11T11:03:28.103Z', + }, + transformations: [], + isConnectionEnabled: true, + isProcessorEnabled: true, + }, + }, + ], + destType: 'hs', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token-hs-additonal-email', + }, + params: {}, + body: { + JSON: { + inputs: [ + { + properties: { lastname: 'Peñarete', firstname: 'Karen' }, + id: '103689', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [{ jobId: 3, userId: 'u1' }], + batched: true, + statusCode: 200, + destination: { + Config: { + authorizationType: 'newPrivateAppApi', + accessToken: 'dummy-access-token-hs-additonal-email', + hubID: 'dummy-hubId', + apiKey: 'dummy-apikey', + apiVersion: 'newApi', + lookupField: 'email', + hubspotEvents: [], + }, + secretConfig: {}, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + name: 'Hubspot', + enabled: true, + workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', + deleted: false, + createdAt: '2020-12-30T08:39:32.005Z', + updatedAt: '2021-02-03T16:22:31.374Z', + destinationDefinition: { + id: '1aIXqM806xAVm92nx07YwKbRrO9', + name: 'HS', + displayName: 'Hubspot', + createdAt: '2020-04-09T09:24:31.794Z', + updatedAt: '2021-01-11T11:03:28.103Z', + }, + transformations: [], + isConnectionEnabled: true, + isProcessorEnabled: true, + }, + }, + ], + }, + }, + }, + }, ];