Merge pull request #258 from w473/allow-fetching-full-export-50

Allow fetching full export 50
pimcore · Apr 12, 2024 · 38e0b72 · 38e0b72
2 parents 02eaa0b + db96fac
commit 38e0b72
Show file tree

Hide file tree

Showing 2 changed files with 62 additions and 11 deletions.
diff --git a/src/Controller/AdminController.php b/src/Controller/AdminController.php
@@ -233,16 +233,11 @@ public function getExportJobsAction(Request $request, Service $service): JsonRes
         $data = json_decode($request->get('filter'), true);
 
         if (empty($ids = $request->get('ids', false))) {
-            $results = $service->doFilter(
+            $ids = $service->getIdsFromFilterNoLimit(
                 $data['classId'],
                 $data['conditions']['filters'],
-                $data['conditions']['fulltextSearchTerm'],
-                0,
-                9999 // elastic search cannot export more results than 9999 in one request
+                $data['conditions']['fulltextSearchTerm']
             );
-
-            //get ID list from ES Service
-            $ids = $service->extractIdsFromResult($results);
         }
 
         $jobs = array_chunk($ids, 20);

diff --git a/src/Service.php b/src/Service.php
@@ -30,6 +30,7 @@
 use ONGR\ElasticsearchDSL\Query\FullText\QueryStringQuery;
 use ONGR\ElasticsearchDSL\Query\TermLevel\WildcardQuery;
 use ONGR\ElasticsearchDSL\Search;
+use ONGR\ElasticsearchDSL\Sort\FieldSort;
 use Pimcore\Db;
 use Pimcore\Model\DataObject\ClassDefinition;
 use Pimcore\Model\DataObject\Concrete;
@@ -795,14 +796,45 @@ public function buildFilterEntryObject($filterEntry)
      * @param string $classId
      * @param array $filters
      * @param BuilderInterface|string $fullTextQuery
-     * @param int $from
-     * @param int $size
      *
-     * @return array|callable
+     * @return array
      *
      * @throws Exception
      */
-    public function doFilter($classId, array $filters, $fullTextQuery, $from = null, $size = null)
+    public function getIdsFromFilterNoLimit($classId, array $filters, $fullTextQuery): array
+    {
+        $search = $this->getDoFilter($classId, $filters, $fullTextQuery);
+        $sort = new FieldSort('id', 'asc');
+
+        $search->addSort($sort);
+
+        $classDefinition = \Pimcore\Model\DataObject\ClassDefinition::getById($classId);
+        $params = [
+            'index' => $this->getIndexName($classDefinition->getName()),
+            'track_total_hits' => true,
+            'rest_total_hits_as_int' => true,
+            'body' => $search->toArray()
+        ];
+
+        $this->logger->info('Filter-Params: ' . json_encode($params));
+
+        $ids = [];
+        do {
+            $results = $this->esClient->search($params)->asArray();
+            $total = $results['hits']['total'];
+            $searchAfter = end($results['hits']['hits'])['sort'];
+            $search->setSearchAfter($searchAfter);
+            $params['body'] = $search->toArray();
+            $ids = array_unique(array_merge($this->extractIdsFromResult($results), $ids));
+
+        } while ($total !== count($ids));
+
+        return $ids;
+    }
+
+
+
+    public function getDoFilter($classId, array $filters, $fullTextQuery): Search
     {
         $classDefinition = \Pimcore\Model\DataObject\ClassDefinition::getById($classId);
 
@@ -816,6 +848,30 @@ public function doFilter($classId, array $filters, $fullTextQuery, $from = null,
 
         $this->eventDispatcher->dispatch(new FilterSearchEvent($search), AdvancedObjectSearchEvents::ELASITIC_FILTER); // @phpstan-ignore-line
 
+        if ($this->user) {
+            $this->addPermissionsExcludeFilter($search);
+        }
+
+        return $search;
+    }
+
+    /**
+     * @param string $classId
+     * @param array $filters
+     * @param BuilderInterface|string $fullTextQuery
+     * @param int $from
+     * @param int $size
+     *
+     * @return array|callable
+     *
+     * @throws Exception
+     */
+    public function doFilter($classId, array $filters, $fullTextQuery, $from = null, $size = null)
+    {
+        $classDefinition = \Pimcore\Model\DataObject\ClassDefinition::getById($classId);
+
+        $search = $this->getDoFilter($classId, $filters, $fullTextQuery);
+
         if ($size) {
             $search->setSize($size);
         }