From d8136e5f636d0b560bba250af6fc8d85f31af03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacek=20G=C5=82ogosz?= Date: Thu, 21 Mar 2024 14:44:03 +0100 Subject: [PATCH 1/2] allow-fetching-full-export v5.0 --- src/Controller/AdminController.php | 9 ++------- src/Service.php | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/Controller/AdminController.php b/src/Controller/AdminController.php index 1e73883..baf70b0 100644 --- a/src/Controller/AdminController.php +++ b/src/Controller/AdminController.php @@ -233,16 +233,11 @@ public function getExportJobsAction(Request $request, Service $service): JsonRes $data = json_decode($request->get('filter'), true); if (empty($ids = $request->get('ids', false))) { - $results = $service->doFilter( + $ids = $service->doFilterNoLimit( $data['classId'], $data['conditions']['filters'], - $data['conditions']['fulltextSearchTerm'], - 0, - 9999 // elastic search cannot export more results than 9999 in one request + $data['conditions']['fulltextSearchTerm'] ); - - //get ID list from ES Service - $ids = $service->extractIdsFromResult($results); } $jobs = array_chunk($ids, 20); diff --git a/src/Service.php b/src/Service.php index 99243ae..a78ee04 100644 --- a/src/Service.php +++ b/src/Service.php @@ -791,6 +791,34 @@ public function buildFilterEntryObject($filterEntry) } } + /** + * @param string $classId + * @param array $filters + * @param BuilderInterface|string $fullTextQuery + * + * @return array + * + * @throws Exception + */ + public function doFilterNoLimit($classId, array $filters, $fullTextQuery): array + { + $from = 0; + $size = 1000; + $ids = []; + + while($results = $this->doFilter( + $classId, + $filters, + $fullTextQuery, + $from, + $size + )){ + $from += $size; + $ids = array_merge($this->extractIdsFromResult($results), $ids); + }; + return $ids; + } + /** * @param string $classId * @param array $filters From db96fac457a95bbfc6ad305105b21bc4f7ab8a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacek=20G=C5=82ogosz?= Date: Fri, 22 Mar 2024 13:44:35 +0100 Subject: [PATCH 2/2] allow fetching full export --- src/Controller/AdminController.php | 2 +- src/Service.php | 72 +++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/src/Controller/AdminController.php b/src/Controller/AdminController.php index baf70b0..9d0e45c 100644 --- a/src/Controller/AdminController.php +++ b/src/Controller/AdminController.php @@ -233,7 +233,7 @@ public function getExportJobsAction(Request $request, Service $service): JsonRes $data = json_decode($request->get('filter'), true); if (empty($ids = $request->get('ids', false))) { - $ids = $service->doFilterNoLimit( + $ids = $service->getIdsFromFilterNoLimit( $data['classId'], $data['conditions']['filters'], $data['conditions']['fulltextSearchTerm'] diff --git a/src/Service.php b/src/Service.php index a78ee04..5d24d93 100644 --- a/src/Service.php +++ b/src/Service.php @@ -30,6 +30,7 @@ use ONGR\ElasticsearchDSL\Query\FullText\QueryStringQuery; use ONGR\ElasticsearchDSL\Query\TermLevel\WildcardQuery; use ONGR\ElasticsearchDSL\Search; +use ONGR\ElasticsearchDSL\Sort\FieldSort; use Pimcore\Db; use Pimcore\Model\DataObject\ClassDefinition; use Pimcore\Model\DataObject\Concrete; @@ -800,25 +801,60 @@ public function buildFilterEntryObject($filterEntry) * * @throws Exception */ - public function doFilterNoLimit($classId, array $filters, $fullTextQuery): array + public function getIdsFromFilterNoLimit($classId, array $filters, $fullTextQuery): array { - $from = 0; - $size = 1000; + $search = $this->getDoFilter($classId, $filters, $fullTextQuery); + $sort = new FieldSort('id', 'asc'); + + $search->addSort($sort); + + $classDefinition = \Pimcore\Model\DataObject\ClassDefinition::getById($classId); + $params = [ + 'index' => $this->getIndexName($classDefinition->getName()), + 'track_total_hits' => true, + 'rest_total_hits_as_int' => true, + 'body' => $search->toArray() + ]; + + $this->logger->info('Filter-Params: ' . json_encode($params)); + $ids = []; + do { + $results = $this->esClient->search($params)->asArray(); + $total = $results['hits']['total']; + $searchAfter = end($results['hits']['hits'])['sort']; + $search->setSearchAfter($searchAfter); + $params['body'] = $search->toArray(); + $ids = array_unique(array_merge($this->extractIdsFromResult($results), $ids)); + + } while ($total !== count($ids)); - while($results = $this->doFilter( - $classId, - $filters, - $fullTextQuery, - $from, - $size - )){ - $from += $size; - $ids = array_merge($this->extractIdsFromResult($results), $ids); - }; return $ids; } + + + public function getDoFilter($classId, array $filters, $fullTextQuery): Search + { + $classDefinition = \Pimcore\Model\DataObject\ClassDefinition::getById($classId); + + $search = $this->getFilter($classDefinition, $filters); + + if ($fullTextQuery instanceof BuilderInterface) { + $search->addQuery($fullTextQuery); + } elseif (!empty($fullTextQuery)) { + $search->addQuery(new QueryStringQuery($fullTextQuery)); + } + + $this->eventDispatcher->dispatch(new FilterSearchEvent($search), AdvancedObjectSearchEvents::ELASITIC_FILTER); // @phpstan-ignore-line + + if ($this->user) { + $this->addPermissionsExcludeFilter($search); + } + + return $search; + } + /** * @param string $classId * @param array $filters @@ -834,15 +870,7 @@ public function doFilter($classId, array $filters, $fullTextQuery, $from = null, { $classDefinition = \Pimcore\Model\DataObject\ClassDefinition::getById($classId); - $search = $this->getFilter($classDefinition, $filters); - - if ($fullTextQuery instanceof BuilderInterface) { - $search->addQuery($fullTextQuery); - } elseif (!empty($fullTextQuery)) { - $search->addQuery(new QueryStringQuery($fullTextQuery)); - } - - $this->eventDispatcher->dispatch(new FilterSearchEvent($search), AdvancedObjectSearchEvents::ELASITIC_FILTER); // @phpstan-ignore-line + $search = $this->getDoFilter($classId, $filters, $fullTextQuery); if ($size) { $search->setSize($size);