diff --git a/fboss/agent/hw/sai/switch/SaiPortManager.cpp b/fboss/agent/hw/sai/switch/SaiPortManager.cpp
index cfa3034758364..e53d90a175c22 100644
--- a/fboss/agent/hw/sai/switch/SaiPortManager.cpp
+++ b/fboss/agent/hw/sai/switch/SaiPortManager.cpp
@@ -896,18 +896,23 @@ SaiPortManager::getIngressPriorityGroupSaiIds(
   return ingressPgSaiIds;
 }
 
-void SaiPortManager::programPfcBuffers(const std::shared_ptr<Port>& swPort) {
+void SaiPortManager::changePfcBuffers(
+    std::shared_ptr<Port> oldPort,
+    std::shared_ptr<Port> newPort) {
   if (!platform_->getAsic()->isSupported(HwAsic::Feature::BUFFER_POOL)) {
     return;
   }
-  SaiPortHandle* portHandle = getPortHandle(swPort->getID());
-  const auto& portPgCfgs = swPort->getPortPgConfigs();
-  if (portPgCfgs) {
-    const auto& ingressPgSaiIds = getIngressPriorityGroupSaiIds(swPort);
+  SaiPortHandle* portHandle = getPortHandle(newPort->getID());
+  auto& configuredIpgs = portHandle->configuredIngressPriorityGroups;
+
+  const auto& newPortPgCfgs = newPort->getPortPgConfigs();
+  std::set<int> programmedPgIds;
+  if (newPortPgCfgs) {
+    const auto& ingressPgSaiIds = getIngressPriorityGroupSaiIds(newPort);
     auto ingressPriorityGroupHandles =
         managerTable_->bufferManager().loadIngressPriorityGroups(
             ingressPgSaiIds);
-    for (const auto& portPgCfg : *portPgCfgs) {
+    for (const auto& portPgCfg : *newPortPgCfgs) {
       // THRIFT_COPY
       auto portPgCfgThrift = portPgCfg->toThrift();
       auto pgId = *portPgCfgThrift.id();
@@ -918,10 +923,32 @@ void SaiPortManager::programPfcBuffers(const std::shared_ptr<Port>& swPort) {
           ingressPriorityGroupHandles[pgId]->ingressPriorityGroup,
           bufferProfile);
       // Keep track of ingressPriorityGroupHandle and bufferProfile per PG ID
-      portHandle
-          ->configuredIngressPriorityGroups[static_cast<IngressPriorityGroupID>(
-              pgId)] = SaiIngressPriorityGroupHandleAndProfile{
-          std::move(ingressPriorityGroupHandles[pgId]), bufferProfile};
+      configuredIpgs[static_cast<IngressPriorityGroupID>(pgId)] =
+          SaiIngressPriorityGroupHandleAndProfile{
+              std::move(ingressPriorityGroupHandles[pgId]), bufferProfile};
+      programmedPgIds.insert(pgId);
+    }
+  }
+
+  // Delete removed buffer profiles.
+  if (oldPort != nullptr) {
+    const auto& oldPortPgCfgs = oldPort->getPortPgConfigs();
+    if (oldPortPgCfgs) {
+      for (const auto& portPgCfg : *oldPortPgCfgs) {
+        // THRIFT_COPY
+        auto portPgCfgThrift = portPgCfg->toThrift();
+        auto pgId = *portPgCfgThrift.id();
+        if (programmedPgIds.find(pgId) == programmedPgIds.end()) {
+          auto ipgInfo =
+              configuredIpgs.find(static_cast<IngressPriorityGroupID>(pgId));
+          if (ipgInfo != configuredIpgs.end()) {
+            managerTable_->bufferManager().setIngressPriorityGroupBufferProfile(
+                ipgInfo->second.pgHandle->ingressPriorityGroup,
+                std::nullptr_t());
+            configuredIpgs.erase(ipgInfo);
+          }
+        }
+      }
     }
   }
 }
diff --git a/fboss/agent/hw/sai/switch/SaiPortManager.h b/fboss/agent/hw/sai/switch/SaiPortManager.h
index 16827826a4e4c..1a7284f604eb7 100644
--- a/fboss/agent/hw/sai/switch/SaiPortManager.h
+++ b/fboss/agent/hw/sai/switch/SaiPortManager.h
@@ -393,7 +393,9 @@ class SaiPortManager {
       const std::shared_ptr<Port>& newPort);
   void removePfcWatchdog(const std::shared_ptr<Port>& swPort);
   void setPortType(PortID portId, cfg::PortType portType);
-  void programPfcBuffers(const std::shared_ptr<Port>& swPort);
+  void changePfcBuffers(
+      std::shared_ptr<Port> oldPort,
+      std::shared_ptr<Port> newPort);
   void removePfcBuffers(const std::shared_ptr<Port>& swPort);
   sai_port_prbs_config_t getSaiPortPrbsConfig(bool enabled) const;
   void initAsicPrbsStats(const std::shared_ptr<Port>& swPort);
diff --git a/fboss/agent/hw/sai/switch/npu/SaiPortManager.cpp b/fboss/agent/hw/sai/switch/npu/SaiPortManager.cpp
index 5a38c5c8cf081..0c917383b7902 100644
--- a/fboss/agent/hw/sai/switch/npu/SaiPortManager.cpp
+++ b/fboss/agent/hw/sai/switch/npu/SaiPortManager.cpp
@@ -271,7 +271,7 @@ PortSaiId SaiPortManager::addPortImpl(const std::shared_ptr<Port>& swPort) {
   addSamplePacket(swPort);
   addNode(swPort);
   addPfc(swPort);
-  programPfcBuffers(swPort);
+  changePfcBuffers(nullptr, swPort);
 
   // set platform port's speed
   auto platformPort = platform_->getPort(swPort->getID());
@@ -355,7 +355,7 @@ void SaiPortManager::changePortImpl(
   changePfc(oldPort, newPort);
   changeRxLaneSquelch(oldPort, newPort);
   changeTxEnable(oldPort, newPort);
-  programPfcBuffers(newPort);
+  changePfcBuffers(oldPort, newPort);
 
   if (newPort->isEnabled()) {
     if (!oldPort->isEnabled()) {