From 92b10f11a590b3299be216326e175c283634f1e6 Mon Sep 17 00:00:00 2001
From: Anant Jain <anantjain45823@gmail.com>
Date: Tue, 26 Dec 2023 12:53:23 +0530
Subject: [PATCH] chore: debug hubspot rate limitng issue

---
 src/util/prometheus.js              |  6 ++++++
 src/v0/destinations/hs/config.js    |  2 +-
 src/v0/destinations/hs/transform.js | 11 +++++++++--
 src/v0/destinations/hs/util.js      |  3 ++-
 4 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/util/prometheus.js b/src/util/prometheus.js
index d7ba3b7c61..dc807f199c 100644
--- a/src/util/prometheus.js
+++ b/src/util/prometheus.js
@@ -310,6 +310,12 @@ class Prometheus {
         type: 'counter',
         labelNames: ['destination', 'success'],
       },
+      {
+        name: 'hs_retl_batch_size',
+        help: 'hs_retl_batch_size',
+        type: 'counter',
+        labelNames: ['destination'],
+      },
       {
         name: 'marketo_bulk_upload_upload_file_jobs',
         help: 'marketo_bulk_upload_upload_file_jobs',
diff --git a/src/v0/destinations/hs/config.js b/src/v0/destinations/hs/config.js
index 3e91035fac..b602a7542f 100644
--- a/src/v0/destinations/hs/config.js
+++ b/src/v0/destinations/hs/config.js
@@ -37,7 +37,7 @@ const IDENTIFY_CRM_UPDATE_CONTACT = `${BASE_ENDPOINT}/crm/v3/objects/contacts/:c
 const BATCH_IDENTIFY_CRM_CREATE_NEW_CONTACT = `${BASE_ENDPOINT}/crm/v3/objects/contacts/batch/create`;
 const BATCH_IDENTIFY_CRM_UPDATE_CONTACT = `${BASE_ENDPOINT}/crm/v3/objects/contacts/batch/update`;
 // Ref - https://developers.hubspot.com/docs/api/crm/contacts#endpoint?spec=GET-/crm/v3/objects/contacts
-const MAX_BATCH_SIZE_CRM_CONTACT = 10;
+const MAX_BATCH_SIZE_CRM_CONTACT = 100;
 
 // Track
 // Ref - https://developers.hubspot.com/docs/api/analytics/events
diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js
index c26e024a6c..f24353b701 100644
--- a/src/v0/destinations/hs/transform.js
+++ b/src/v0/destinations/hs/transform.js
@@ -7,6 +7,7 @@ const {
   getDestinationExternalIDInfoForRetl,
 } = require('../../util');
 const { API_VERSION } = require('./config');
+const stats = require('../../../util/stats');
 const {
   processLegacyIdentify,
   processLegacyTrack,
@@ -86,11 +87,17 @@ const processRouterDest = async (inputs, reqMetadata) => {
 
   try {
     if (mappedToDestination && GENERIC_TRUE_VALUES.includes(mappedToDestination?.toString())) {
+      // retl
+      // fire metric to get batch size
+      stats.histogram('hs_retl_batch_size', tempInputs.length, {
+        destination: 'HUBSPOT',
+      });
+
       // skip splitting the batches to inserts and updates if object it is an association
       if (objectType.toLowerCase() !== 'association') {
-        propertyMap = await getProperties(destination);
+        propertyMap = await getProperties(destination); // 1 for 100 events
         // get info about existing objects and splitting accordingly.
-        tempInputs = await splitEventsForCreateUpdate(tempInputs, destination);
+        tempInputs = await splitEventsForCreateUpdate(tempInputs, destination); // 1
       }
     } else {
       // reduce the no. of calls for properties endpoint
diff --git a/src/v0/destinations/hs/util.js b/src/v0/destinations/hs/util.js
index f83ce0b6de..0f80583d85 100644
--- a/src/v0/destinations/hs/util.js
+++ b/src/v0/destinations/hs/util.js
@@ -538,7 +538,8 @@ const getExistingData = async (inputs, destination) => {
             feature: 'transformation',
             endpointPath,
           })
-        : await httpPOST(url, requestData, {
+        : await httpPOST(url, requestData, { 
+          // 100 REQ -> 100/10 + 1 + 1 -> 12
             destType: 'hs',
             feature: 'transformation',
             endpointPath,