From d5bf59fb5720beaa1fd38cc3d445ee29941f5662 Mon Sep 17 00:00:00 2001 From: Benjamin Gaussorgues Date: Thu, 10 Oct 2024 09:48:53 +0200 Subject: [PATCH] feat(dav): introduce paginate with custom headers Signed-off-by: Benjamin Gaussorgues --- apps/dav/appinfo/info.xml | 3 +- .../composer/composer/autoload_classmap.php | 6 ++ .../dav/composer/composer/autoload_static.php | 6 ++ .../BackgroundJob/CleanupPaginateCacheJob.php | 26 +++++ .../Version1032Date20241011093632.php | 63 ++++++++++++ apps/dav/lib/Paginate/LimitedCopyIterator.php | 49 ++++++++++ apps/dav/lib/Paginate/PaginateCache.php | 98 +++++++++++++++++++ apps/dav/lib/Paginate/PaginatePlugin.php | 96 ++++++++++++++++++ apps/dav/lib/Server.php | 2 + 9 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 apps/dav/lib/BackgroundJob/CleanupPaginateCacheJob.php create mode 100644 apps/dav/lib/Migration/Version1032Date20241011093632.php create mode 100644 apps/dav/lib/Paginate/LimitedCopyIterator.php create mode 100644 apps/dav/lib/Paginate/PaginateCache.php create mode 100644 apps/dav/lib/Paginate/PaginatePlugin.php diff --git a/apps/dav/appinfo/info.xml b/apps/dav/appinfo/info.xml index 6f5a085ef0576..854dcf92d80c2 100644 --- a/apps/dav/appinfo/info.xml +++ b/apps/dav/appinfo/info.xml @@ -10,7 +10,7 @@ WebDAV WebDAV endpoint WebDAV endpoint - 1.32.0 + 1.33.0 agpl owncloud.org DAV @@ -27,6 +27,7 @@ OCA\DAV\BackgroundJob\CleanupDirectLinksJob OCA\DAV\BackgroundJob\UpdateCalendarResourcesRoomsBackgroundJob OCA\DAV\BackgroundJob\CleanupInvitationTokenJob + OCA\DAV\BackgroundJob\CleanupPaginateCacheJob OCA\DAV\BackgroundJob\EventReminderJob OCA\DAV\BackgroundJob\CalendarRetentionJob OCA\DAV\BackgroundJob\PruneOutdatedSyncTokensJob diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php index 9e8cd76d0606a..238987a651ba0 100644 --- a/apps/dav/composer/composer/autoload_classmap.php +++ b/apps/dav/composer/composer/autoload_classmap.php @@ -16,6 +16,7 @@ 'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => $baseDir . '/../lib/BackgroundJob/CalendarRetentionJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => $baseDir . '/../lib/BackgroundJob/CleanupDirectLinksJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php', + 'OCA\\DAV\\BackgroundJob\\CleanupPaginateCacheJob' => $baseDir . '/../lib/BackgroundJob/CleanupPaginateCacheJob.php', 'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => $baseDir . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php', 'OCA\\DAV\\BackgroundJob\\EventReminderJob' => $baseDir . '/../lib/BackgroundJob/EventReminderJob.php', 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', @@ -328,6 +329,7 @@ 'OCA\\DAV\\Migration\\Version1008Date20181105110300' => $baseDir . '/../lib/Migration/Version1008Date20181105110300.php', 'OCA\\DAV\\Migration\\Version1008Date20181105112049' => $baseDir . '/../lib/Migration/Version1008Date20181105112049.php', 'OCA\\DAV\\Migration\\Version1008Date20181114084440' => $baseDir . '/../lib/Migration/Version1008Date20181114084440.php', + 'OCA\\DAV\\Migration\\Version1009Date20181108161232' => $baseDir . '/../lib/Migration/Version1009Date20181108161232.php', 'OCA\\DAV\\Migration\\Version1011Date20190725113607' => $baseDir . '/../lib/Migration/Version1011Date20190725113607.php', 'OCA\\DAV\\Migration\\Version1011Date20190806104428' => $baseDir . '/../lib/Migration/Version1011Date20190806104428.php', 'OCA\\DAV\\Migration\\Version1012Date20190808122342' => $baseDir . '/../lib/Migration/Version1012Date20190808122342.php', @@ -341,6 +343,10 @@ 'OCA\\DAV\\Migration\\Version1029Date20231004091403' => $baseDir . '/../lib/Migration/Version1029Date20231004091403.php', 'OCA\\DAV\\Migration\\Version1030Date20240205103243' => $baseDir . '/../lib/Migration/Version1030Date20240205103243.php', 'OCA\\DAV\\Migration\\Version1031Date20240610134258' => $baseDir . '/../lib/Migration/Version1031Date20240610134258.php', + 'OCA\\DAV\\Migration\\Version1032Date20241011093632' => $baseDir . '/../lib/Migration/Version1032Date20241011093632.php', + 'OCA\\DAV\\Paginate\\LimitedCopyIterator' => $baseDir . '/../lib/Paginate/LimitedCopyIterator.php', + 'OCA\\DAV\\Paginate\\PaginateCache' => $baseDir . '/../lib/Paginate/PaginateCache.php', + 'OCA\\DAV\\Paginate\\PaginatePlugin' => $baseDir . '/../lib/Paginate/PaginatePlugin.php', 'OCA\\DAV\\Profiler\\ProfilerPlugin' => $baseDir . '/../lib/Profiler/ProfilerPlugin.php', 'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningNode.php', 'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php', diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php index fc1838c1b4fbe..e969ac5a592e4 100644 --- a/apps/dav/composer/composer/autoload_static.php +++ b/apps/dav/composer/composer/autoload_static.php @@ -31,6 +31,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CalendarRetentionJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupDirectLinksJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php', + 'OCA\\DAV\\BackgroundJob\\CleanupPaginateCacheJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupPaginateCacheJob.php', 'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => __DIR__ . '/..' . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php', 'OCA\\DAV\\BackgroundJob\\EventReminderJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/EventReminderJob.php', 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', @@ -343,6 +344,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\Migration\\Version1008Date20181105110300' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181105110300.php', 'OCA\\DAV\\Migration\\Version1008Date20181105112049' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181105112049.php', 'OCA\\DAV\\Migration\\Version1008Date20181114084440' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181114084440.php', + 'OCA\\DAV\\Migration\\Version1009Date20181108161232' => __DIR__ . '/..' . '/../lib/Migration/Version1009Date20181108161232.php', 'OCA\\DAV\\Migration\\Version1011Date20190725113607' => __DIR__ . '/..' . '/../lib/Migration/Version1011Date20190725113607.php', 'OCA\\DAV\\Migration\\Version1011Date20190806104428' => __DIR__ . '/..' . '/../lib/Migration/Version1011Date20190806104428.php', 'OCA\\DAV\\Migration\\Version1012Date20190808122342' => __DIR__ . '/..' . '/../lib/Migration/Version1012Date20190808122342.php', @@ -356,6 +358,10 @@ class ComposerStaticInitDAV 'OCA\\DAV\\Migration\\Version1029Date20231004091403' => __DIR__ . '/..' . '/../lib/Migration/Version1029Date20231004091403.php', 'OCA\\DAV\\Migration\\Version1030Date20240205103243' => __DIR__ . '/..' . '/../lib/Migration/Version1030Date20240205103243.php', 'OCA\\DAV\\Migration\\Version1031Date20240610134258' => __DIR__ . '/..' . '/../lib/Migration/Version1031Date20240610134258.php', + 'OCA\\DAV\\Migration\\Version1032Date20241011093632' => __DIR__ . '/..' . '/../lib/Migration/Version1032Date20241011093632.php', + 'OCA\\DAV\\Paginate\\LimitedCopyIterator' => __DIR__ . '/..' . '/../lib/Paginate/LimitedCopyIterator.php', + 'OCA\\DAV\\Paginate\\PaginateCache' => __DIR__ . '/..' . '/../lib/Paginate/PaginateCache.php', + 'OCA\\DAV\\Paginate\\PaginatePlugin' => __DIR__ . '/..' . '/../lib/Paginate/PaginatePlugin.php', 'OCA\\DAV\\Profiler\\ProfilerPlugin' => __DIR__ . '/..' . '/../lib/Profiler/ProfilerPlugin.php', 'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningNode.php', 'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php', diff --git a/apps/dav/lib/BackgroundJob/CleanupPaginateCacheJob.php b/apps/dav/lib/BackgroundJob/CleanupPaginateCacheJob.php new file mode 100644 index 0000000000000..62f5b30f01302 --- /dev/null +++ b/apps/dav/lib/BackgroundJob/CleanupPaginateCacheJob.php @@ -0,0 +1,26 @@ +cache = $cache; + } + + public function run($argument): void { + $this->cache->cleanup(); + } +} diff --git a/apps/dav/lib/Migration/Version1032Date20241011093632.php b/apps/dav/lib/Migration/Version1032Date20241011093632.php new file mode 100644 index 0000000000000..8618b9c6dd19b --- /dev/null +++ b/apps/dav/lib/Migration/Version1032Date20241011093632.php @@ -0,0 +1,63 @@ +hasTable('dav_page_cache')) { + $table = $schema->createTable('dav_page_cache'); + + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true + ]); + $table->addColumn('url_hash', Types::STRING, [ + 'notnull' => true, + 'length' => 32, + ]); + $table->addColumn('token', Types::STRING, [ + 'notnull' => true, + 'length' => 32 + ]); + $table->addColumn('result_index', Types::INTEGER, [ + 'notnull' => true + ]); + $table->addColumn('result_value', Types::TEXT, [ + 'notnull' => false, + ]); + $table->addColumn('insert_time', Types::DATETIME, [ + 'notnull' => true, + ]); + + $table->setPrimaryKey(['id'], 'dav_page_cache_id_index'); + $table->addIndex(['token', 'url_hash'], 'dav_page_cache_token_url'); + $table->addUniqueIndex(['token', 'url_hash', 'result_index'], 'dav_page_cache_url_index'); + $table->addIndex(['result_index'], 'dav_page_cache_index'); + $table->addIndex(['insert_time'], 'dav_page_cache_time'); + } + + return $schema; + } +} diff --git a/apps/dav/lib/Paginate/LimitedCopyIterator.php b/apps/dav/lib/Paginate/LimitedCopyIterator.php new file mode 100644 index 0000000000000..8cbd77e3be19c --- /dev/null +++ b/apps/dav/lib/Paginate/LimitedCopyIterator.php @@ -0,0 +1,49 @@ +valid() && ++$i <= $offset) { + $this->skipped[] = $iterator->current(); + $iterator->next(); + } + + while ($iterator->valid() && count($this->copy) < $count) { + $this->copy[] = $iterator->current(); + $iterator->next(); + } + + $this->append(new \ArrayIterator($this->skipped)); + $this->append($this->getRequestedItems()); + $this->append($iterator); + } + + public function getRequestedItems(): \Iterator { + return new \ArrayIterator($this->copy); + } +} diff --git a/apps/dav/lib/Paginate/PaginateCache.php b/apps/dav/lib/Paginate/PaginateCache.php new file mode 100644 index 0000000000000..2d8114c2115d5 --- /dev/null +++ b/apps/dav/lib/Paginate/PaginateCache.php @@ -0,0 +1,98 @@ +random->generate(32); + $now = $this->timeFactory->getDateTime(); + + $query = $this->database->getQueryBuilder(); + $query->insert('dav_page_cache') + ->values([ + 'url_hash' => $query->createNamedParameter(md5($uri), IQueryBuilder::PARAM_STR), + 'token' => $query->createNamedParameter($token, IQueryBuilder::PARAM_STR), + 'insert_time' => $query->createNamedParameter($now, IQueryBuilder::PARAM_DATETIME_MUTABLE), + 'result_index' => $query->createParameter('index'), + 'result_value' => $query->createParameter('value'), + ]); + + $count = 0; + foreach ($items as $item) { + $value = serialize($item); + $query->setParameter('index', $count, IQueryBuilder::PARAM_INT); + $query->setParameter('value', $value); + $query->executeStatement(); + $count++; + } + + return ['token' => $token, 'count' => $count]; + } + + public function get(string $url, string $token, int $offset, int $count): array { + $query = $this->database->getQueryBuilder(); + $query->select(['result_value']) + ->from('dav_page_cache') + ->where($query->expr()->eq('token', $query->createNamedParameter($token))) + ->andWhere($query->expr()->eq('url_hash', $query->createNamedParameter(md5($url)))) + ->andWhere($query->expr()->gte('result_index', $query->createNamedParameter($offset, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->lt('result_index', $query->createNamedParameter($offset + $count, IQueryBuilder::PARAM_INT))); + + $result = $query->executeQuery(); + return array_map(function (string $entry) { + return unserialize($entry); + }, $result->fetchAll(\PDO::FETCH_COLUMN)); + } + + public function exists(string $token): bool { + $query = $this->database->getQueryBuilder(); + return (bool)$query->select('id') + ->from('dav_page_cache') + ->where($query->expr()->eq('token', $query->createNamedParameter($token))) + ->setMaxResults(1) + ->executeQuery() + ->fetchOne(); + } + + public function cleanup(): void { + $now = $this->timeFactory->getDateTime(); + $minDate = $now->sub(\DateInterval::createFromDateString(self::TTL)); + + $query = $this->database->getQueryBuilder(); + $query->delete('dav_page_cache') + ->where($query->expr()->lt('insert_time', $query->createNamedParameter($minDate, IQueryBuilder::PARAM_DATETIME_MUTABLE))); + $query->executeStatement(); + } + + public function clear(): void { + $query = $this->database->getQueryBuilder(); + $query->delete('dav_page_cache'); + $query->executeStatement(); + } +} diff --git a/apps/dav/lib/Paginate/PaginatePlugin.php b/apps/dav/lib/Paginate/PaginatePlugin.php new file mode 100644 index 0000000000000..e09d8e2c87a62 --- /dev/null +++ b/apps/dav/lib/Paginate/PaginatePlugin.php @@ -0,0 +1,96 @@ +server = $server; + $server->on('beforeMultiStatus', [$this, 'onMultiStatus']); + $server->on('method:SEARCH', [$this, 'onMethod'], 1); + $server->on('method:PROPFIND', [$this, 'onMethod'], 1); + $server->on('method:REPORT', [$this, 'onMethod'], 1); + } + + public function getFeatures(): array { + return ['nc-paginate']; + } + + public function onMultiStatus(&$fileProperties): void { + $request = $this->server->httpRequest; + if (is_array($fileProperties)) { + $fileProperties = new \ArrayIterator($fileProperties); + } + if ( + $request->hasHeader(self::PAGINATE_HEADER) && + (!$request->hasHeader(self::PAGINATE_TOKEN_HEADER) || !$this->cache->exists($request->getHeader(self::PAGINATE_TOKEN_HEADER))) + ) { + $url = $request->getUrl(); + + $pageSize = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize; + $offset = (int)$request->getHeader(self::PAGINATE_OFFSET_HEADER); + $copyIterator = new LimitedCopyIterator($fileProperties, $pageSize, $offset); + ['token' => $token, 'count' => $count] = $this->cache->store($url, $copyIterator); + + $fileProperties = $copyIterator->getRequestedItems(); + $this->server->httpResponse->addHeader(self::PAGINATE_HEADER, 'true'); + $this->server->httpResponse->addHeader(self::PAGINATE_TOKEN_HEADER, $token); + $this->server->httpResponse->addHeader(self::PAGINATE_TOTAL_HEADER, (string)$count); + $request->setHeader(self::PAGINATE_TOKEN_HEADER, $token); + } + } + + public function onMethod(RequestInterface $request, ResponseInterface $response) { + if ( + $request->hasHeader(self::PAGINATE_TOKEN_HEADER) && + $request->hasHeader(self::PAGINATE_OFFSET_HEADER) && + $this->cache->exists($request->getHeader(self::PAGINATE_TOKEN_HEADER)) + ) { + $url = $this->server->httpRequest->getUrl(); + $token = $request->getHeader(self::PAGINATE_TOKEN_HEADER); + $offset = (int)$request->getHeader(self::PAGINATE_OFFSET_HEADER); + $count = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize; + + $items = $this->cache->get($url, $token, $offset, $count); + + $response->setStatus(207); + $response->addHeader(self::PAGINATE_HEADER, 'true'); + $response->setHeader('Content-Type', 'application/xml; charset=utf-8'); + $response->setHeader('Vary', 'Brief,Prefer'); + + $prefer = $this->server->getHTTPPrefer(); + $minimal = $prefer['return'] === 'minimal'; + + $data = $this->server->generateMultiStatus($items, $minimal); + $response->setBody($data); + + return false; + } + } +} diff --git a/apps/dav/lib/Server.php b/apps/dav/lib/Server.php index 0dfdd43bf0cb9..698e5147374b7 100644 --- a/apps/dav/lib/Server.php +++ b/apps/dav/lib/Server.php @@ -57,6 +57,7 @@ use OCA\DAV\Files\ErrorPagePlugin; use OCA\DAV\Files\FileSearchBackend; use OCA\DAV\Files\LazySearchBackend; +use OCA\DAV\Paginate\PaginatePlugin; use OCA\DAV\Profiler\ProfilerPlugin; use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin; use OCA\DAV\SystemTag\SystemTagPlugin; @@ -228,6 +229,7 @@ public function __construct( $logger, $eventDispatcher, )); + $this->server->addPlugin(\OCP\Server::get(PaginatePlugin::class)); // allow setup of additional plugins $eventDispatcher->dispatch('OCA\DAV\Connector\Sabre::addPlugin', $event);