From dd1ad66d8f17a215fd5f01f1108dae47846881fa Mon Sep 17 00:00:00 2001
From: Divesh Pahuja <divesh.pahuja@pimcore.com>
Date: Tue, 26 Sep 2023 17:41:53 +0200
Subject: [PATCH 1/3] Fix too many connections error on import - resolves #354

---
 src/Command/ParallelProcessQueueCommand.php   | 5 +++++
 src/Command/SequentialProcessQueueCommand.php | 7 ++++++-
 src/Messenger/DataImporterHandler.php         | 7 ++++++-
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/Command/ParallelProcessQueueCommand.php b/src/Command/ParallelProcessQueueCommand.php
index 11347a1b..b4077c68 100644
--- a/src/Command/ParallelProcessQueueCommand.php
+++ b/src/Command/ParallelProcessQueueCommand.php
@@ -68,5 +68,10 @@ protected function doFetchItems(InputInterface $input, ?OutputInterface $output)
     protected function runSingleCommand(string $item, InputInterface $input, OutputInterface $output): void
     {
         $this->importProcessingService->processQueueItem((int) $item);
+
+        // call the garbage collector if memory consumption is > 100MB
+        if (memory_get_usage() > 100000000) {
+            \Pimcore::collectGarbage();
+        }
     }
 }
diff --git a/src/Command/SequentialProcessQueueCommand.php b/src/Command/SequentialProcessQueueCommand.php
index eb95021e..acdb694d 100644
--- a/src/Command/SequentialProcessQueueCommand.php
+++ b/src/Command/SequentialProcessQueueCommand.php
@@ -83,9 +83,14 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $progressBar = new ProgressBar($output, $itemCount);
         $progressBar->start();
 
-        foreach ($itemIds as $id) {
+        foreach ($itemIds as $i => $id) {
             $this->importProcessingService->processQueueItem($id);
             $progressBar->advance();
+
+            // call the garbage collector to avoid too many connections & memory issue
+            if (($i + 1) % 200 === 0) {
+                \Pimcore::collectGarbage();
+            }
         }
 
         $progressBar->finish();
diff --git a/src/Messenger/DataImporterHandler.php b/src/Messenger/DataImporterHandler.php
index a25c79f9..98a28707 100644
--- a/src/Messenger/DataImporterHandler.php
+++ b/src/Messenger/DataImporterHandler.php
@@ -50,8 +50,13 @@ public function __construct(
 
     public function __invoke(DataImporterMessage $message)
     {
-        foreach ($message->getIds() as $id) {
+        foreach ($message->getIds() as $i => $id) {
             $this->importProcessingService->processQueueItem($id);
+
+            // call the garbage collector to avoid too many connections & memory issue
+            if (($i + 1) % 200 === 0) {
+                \Pimcore::collectGarbage();
+            }
         }
 
         $this->removeMessage($message->getMessageId());

From 02e9b33d731e9290bd2d8e53b07c7c06bde84ba7 Mon Sep 17 00:00:00 2001
From: Divesh Pahuja <divesh.pahuja@pimcore.com>
Date: Mon, 9 Oct 2023 11:34:23 +0200
Subject: [PATCH 2/3] Review changes

---
 src/Command/ParallelProcessQueueCommand.php | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/Command/ParallelProcessQueueCommand.php b/src/Command/ParallelProcessQueueCommand.php
index b4077c68..11347a1b 100644
--- a/src/Command/ParallelProcessQueueCommand.php
+++ b/src/Command/ParallelProcessQueueCommand.php
@@ -68,10 +68,5 @@ protected function doFetchItems(InputInterface $input, ?OutputInterface $output)
     protected function runSingleCommand(string $item, InputInterface $input, OutputInterface $output): void
     {
         $this->importProcessingService->processQueueItem((int) $item);
-
-        // call the garbage collector if memory consumption is > 100MB
-        if (memory_get_usage() > 100000000) {
-            \Pimcore::collectGarbage();
-        }
     }
 }

From 0fd63ae47f91d2a7747a5ba1f8b1a12f45ec9048 Mon Sep 17 00:00:00 2001
From: Divesh Pahuja <divesh.pahuja@pimcore.com>
Date: Thu, 12 Oct 2023 11:51:16 +0200
Subject: [PATCH 3/3] Review changes

---
 src/Messenger/DataImporterHandler.php | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/Messenger/DataImporterHandler.php b/src/Messenger/DataImporterHandler.php
index 98a28707..a25c79f9 100644
--- a/src/Messenger/DataImporterHandler.php
+++ b/src/Messenger/DataImporterHandler.php
@@ -50,13 +50,8 @@ public function __construct(
 
     public function __invoke(DataImporterMessage $message)
     {
-        foreach ($message->getIds() as $i => $id) {
+        foreach ($message->getIds() as $id) {
             $this->importProcessingService->processQueueItem($id);
-
-            // call the garbage collector to avoid too many connections & memory issue
-            if (($i + 1) % 200 === 0) {
-                \Pimcore::collectGarbage();
-            }
         }
 
         $this->removeMessage($message->getMessageId());