chore: debug hubspot rate limitng issue

rudderlabs · Dec 26, 2023 · 92b10f1 · 92b10f1
1 parent 4c26720
commit 92b10f1
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 4 deletions.
diff --git a/src/util/prometheus.js b/src/util/prometheus.js
@@ -310,6 +310,12 @@ class Prometheus {
         type: 'counter',
         labelNames: ['destination', 'success'],
       },
+      {
+        name: 'hs_retl_batch_size',
+        help: 'hs_retl_batch_size',
+        type: 'counter',
+        labelNames: ['destination'],
+      },
       {
         name: 'marketo_bulk_upload_upload_file_jobs',
         help: 'marketo_bulk_upload_upload_file_jobs',

diff --git a/src/v0/destinations/hs/config.js b/src/v0/destinations/hs/config.js
@@ -37,7 +37,7 @@ const IDENTIFY_CRM_UPDATE_CONTACT = `${BASE_ENDPOINT}/crm/v3/objects/contacts/:c
 const BATCH_IDENTIFY_CRM_CREATE_NEW_CONTACT = `${BASE_ENDPOINT}/crm/v3/objects/contacts/batch/create`;
 const BATCH_IDENTIFY_CRM_UPDATE_CONTACT = `${BASE_ENDPOINT}/crm/v3/objects/contacts/batch/update`;
 // Ref - https://developers.hubspot.com/docs/api/crm/contacts#endpoint?spec=GET-/crm/v3/objects/contacts
-const MAX_BATCH_SIZE_CRM_CONTACT = 10;
+const MAX_BATCH_SIZE_CRM_CONTACT = 100;
 
 // Track
 // Ref - https://developers.hubspot.com/docs/api/analytics/events

diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js
@@ -7,6 +7,7 @@ const {
   getDestinationExternalIDInfoForRetl,
 } = require('../../util');
 const { API_VERSION } = require('./config');
+const stats = require('../../../util/stats');
 const {
   processLegacyIdentify,
   processLegacyTrack,
@@ -86,11 +87,17 @@ const processRouterDest = async (inputs, reqMetadata) => {
 
   try {
     if (mappedToDestination && GENERIC_TRUE_VALUES.includes(mappedToDestination?.toString())) {
+      // retl
+      // fire metric to get batch size
+      stats.histogram('hs_retl_batch_size', tempInputs.length, {
+        destination: 'HUBSPOT',
+      });
+
       // skip splitting the batches to inserts and updates if object it is an association
       if (objectType.toLowerCase() !== 'association') {
-        propertyMap = await getProperties(destination);
+        propertyMap = await getProperties(destination); // 1 for 100 events
         // get info about existing objects and splitting accordingly.
-        tempInputs = await splitEventsForCreateUpdate(tempInputs, destination);
+        tempInputs = await splitEventsForCreateUpdate(tempInputs, destination); // 1
       }
     } else {
       // reduce the no. of calls for properties endpoint

diff --git a/src/v0/destinations/hs/util.js b/src/v0/destinations/hs/util.js
@@ -538,7 +538,8 @@ const getExistingData = async (inputs, destination) => {
             feature: 'transformation',
             endpointPath,
           })
-        : await httpPOST(url, requestData, {
+        : await httpPOST(url, requestData, { 
+          // 100 REQ -> 100/10 + 1 + 1 -> 12
             destType: 'hs',
             feature: 'transformation',
             endpointPath,