Merge pull request #48662 from nextcloud/feat/dav-pagination

feat(dav): introduce paginate with custom headers

apps/dav/appinfo/info.xml

@@ -10,7 +10,7 @@
 	<name>WebDAV</name>
 	<summary>WebDAV endpoint</summary>
 	<description>WebDAV endpoint</description>
-	<version>1.32.0</version>
+	<version>1.33.0</version>
 	<licence>agpl</licence>
 	<author>owncloud.org</author>
 	<namespace>DAV</namespace>

apps/dav/composer/composer/autoload_classmap.php

@@ -342,6 +342,9 @@
     'OCA\\DAV\\Migration\\Version1029Date20231004091403' => $baseDir . '/../lib/Migration/Version1029Date20231004091403.php',
     'OCA\\DAV\\Migration\\Version1030Date20240205103243' => $baseDir . '/../lib/Migration/Version1030Date20240205103243.php',
     'OCA\\DAV\\Migration\\Version1031Date20240610134258' => $baseDir . '/../lib/Migration/Version1031Date20240610134258.php',
+    'OCA\\DAV\\Paginate\\LimitedCopyIterator' => $baseDir . '/../lib/Paginate/LimitedCopyIterator.php',
+    'OCA\\DAV\\Paginate\\PaginateCache' => $baseDir . '/../lib/Paginate/PaginateCache.php',
+    'OCA\\DAV\\Paginate\\PaginatePlugin' => $baseDir . '/../lib/Paginate/PaginatePlugin.php',
     'OCA\\DAV\\Profiler\\ProfilerPlugin' => $baseDir . '/../lib/Profiler/ProfilerPlugin.php',
     'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningNode.php',
     'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php',

apps/dav/composer/composer/autoload_static.php

@@ -357,6 +357,9 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Migration\\Version1029Date20231004091403' => __DIR__ . '/..' . '/../lib/Migration/Version1029Date20231004091403.php',
         'OCA\\DAV\\Migration\\Version1030Date20240205103243' => __DIR__ . '/..' . '/../lib/Migration/Version1030Date20240205103243.php',
         'OCA\\DAV\\Migration\\Version1031Date20240610134258' => __DIR__ . '/..' . '/../lib/Migration/Version1031Date20240610134258.php',
+        'OCA\\DAV\\Paginate\\LimitedCopyIterator' => __DIR__ . '/..' . '/../lib/Paginate/LimitedCopyIterator.php',
+        'OCA\\DAV\\Paginate\\PaginateCache' => __DIR__ . '/..' . '/../lib/Paginate/PaginateCache.php',
+        'OCA\\DAV\\Paginate\\PaginatePlugin' => __DIR__ . '/..' . '/../lib/Paginate/PaginatePlugin.php',
         'OCA\\DAV\\Profiler\\ProfilerPlugin' => __DIR__ . '/..' . '/../lib/Profiler/ProfilerPlugin.php',
         'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningNode.php',
         'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php',

apps/dav/lib/Paginate/LimitedCopyIterator.php

@@ -0,0 +1,51 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\DAV\Paginate;
+
+/**
+ * Save a copy of the first X items into a separate iterator
+ *
+ * This allows us to pass the iterator to the cache while keeping a copy
+ * of the required items.
+ *
+ * @extends \AppendIterator<int, int, \Iterator<int, int>>
+ */
+class LimitedCopyIterator extends \AppendIterator {
+	private array $skipped = [];
+	private array $copy = [];
+
+	public function __construct(\Traversable $iterator, int $count, int $offset = 0) {
+		parent::__construct();
+
+		if (!$iterator instanceof \Iterator) {
+			$iterator = new \IteratorIterator($iterator);
+		}
+		$iterator = new \NoRewindIterator($iterator);
+
+		$i = 0;
+		while ($iterator->valid() && ++$i <= $offset) {
+			$this->skipped[] = $iterator->current();
+			$iterator->next();
+		}
+
+		while ($iterator->valid() && count($this->copy) < $count) {
+			$this->copy[] = $iterator->current();
+			$iterator->next();
+		}
+
+		$this->append(new \ArrayIterator($this->skipped));
+		$this->append($this->getRequestedItems());
+		$this->append($iterator);
+	}
+
+	public function getRequestedItems(): \Iterator {
+		return new \ArrayIterator($this->copy);
+	}
+}

apps/dav/lib/Paginate/PaginateCache.php

@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\DAV\Paginate;
+
+use Generator;
+use OCP\ICache;
+use OCP\ICacheFactory;
+use OCP\IDBConnection;
+use OCP\Security\ISecureRandom;
+
+class PaginateCache {
+	public const TTL = 60 * 60;
+	private const CACHE_COUNT_SUFFIX = 'count';
+
+	private ICache $cache;
+
+	public function __construct(
+		private IDBConnection $database,
+		private ISecureRandom $random,
+		ICacheFactory $cacheFactory,
+	) {
+		$this->cache = $cacheFactory->createDistributed('pagination_');
+	}
+
+	/**
+	 * @param string $uri
+	 * @param \Iterator $items
+	 * @return array{'token': string, 'count': int}
+	 */
+	public function store(string $uri, \Iterator $items): array {
+		$token = $this->random->generate(32);
+		$cacheKey = $this->buildCacheKey($uri, $token);
+
+		$count = 0;
+		foreach ($items as $item) {
+			// Add small margin to avoid fetching valid count and then expired entries
+			$this->cache->set($cacheKey . $count, $item, self::TTL + 60);
+			++$count;
+		}
+		$this->cache->set($cacheKey . self::CACHE_COUNT_SUFFIX, $count, self::TTL);
+
+		return ['token' => $token, 'count' => $count];
+	}
+
+	/**
+	 * @return Generator<mixed>
+	 */
+	public function get(string $uri, string $token, int $offset, int $count): Generator {
+		$cacheKey = $this->buildCacheKey($uri, $token);
+		$nbItems = $this->cache->get($cacheKey . self::CACHE_COUNT_SUFFIX);
+		if (!$nbItems || $offset > $nbItems) {
+			return [];
+		}
+
+		$lastItem = min($nbItems, $offset + $count);
+		for ($i = $offset; $i < $lastItem; ++$i) {
+			yield $this->cache->get($cacheKey . $i);
+		}
+	}
+
+	public function exists(string $uri, string $token): bool {
+		return $this->cache->get($this->buildCacheKey($uri, $token) . self::CACHE_COUNT_SUFFIX) > 0;
+	}
+
+	private function buildCacheKey(string $uri, string $token): string {
+		return $token . '_' . crc32($uri) . '_';
+	}
+}

apps/dav/lib/Paginate/PaginatePlugin.php

@@ -0,0 +1,95 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+namespace OCA\DAV\Paginate;
+
+use Sabre\DAV\Server;
+use Sabre\DAV\ServerPlugin;
+use Sabre\HTTP\RequestInterface;
+use Sabre\HTTP\ResponseInterface;
+
+class PaginatePlugin extends ServerPlugin {
+	public const PAGINATE_HEADER = 'X-NC-Paginate';
+	public const PAGINATE_TOTAL_HEADER = 'X-NC-Paginate-Total';
+	public const PAGINATE_TOKEN_HEADER = 'X-NC-Paginate-Token';
+	public const PAGINATE_OFFSET_HEADER = 'X-NC-Paginate-Offset';
+	public const PAGINATE_COUNT_HEADER = 'X-NC-Paginate-Count';
+
+	/** @var Server */
+	private $server;
+
+	public function __construct(
+		private PaginateCache $cache,
+		private int $pageSize = 100,
+	) {
+	}
+
+	public function initialize(Server $server): void {
+		$this->server = $server;
+		$server->on('beforeMultiStatus', [$this, 'onMultiStatus']);
+		$server->on('method:SEARCH', [$this, 'onMethod'], 1);
+		$server->on('method:PROPFIND', [$this, 'onMethod'], 1);
+		$server->on('method:REPORT', [$this, 'onMethod'], 1);
+	}
+
+	public function getFeatures(): array {
+		return ['nc-paginate'];
+	}
+
+	public function onMultiStatus(&$fileProperties): void {
+		$request = $this->server->httpRequest;
+		if (is_array($fileProperties)) {
+			$fileProperties = new \ArrayIterator($fileProperties);
+		}
+		$url = $request->getUrl();
+		if (
+			$request->hasHeader(self::PAGINATE_HEADER) &&
+			(!$request->hasHeader(self::PAGINATE_TOKEN_HEADER) || !$this->cache->exists($url, $request->getHeader(self::PAGINATE_TOKEN_HEADER)))
+		) {
+			$pageSize = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize;
+			$offset = (int)$request->getHeader(self::PAGINATE_OFFSET_HEADER);
+			$copyIterator = new LimitedCopyIterator($fileProperties, $pageSize, $offset);
+			['token' => $token, 'count' => $count] = $this->cache->store($url, $copyIterator);
+
+			$fileProperties = $copyIterator->getRequestedItems();
+			$this->server->httpResponse->addHeader(self::PAGINATE_HEADER, 'true');
+			$this->server->httpResponse->addHeader(self::PAGINATE_TOKEN_HEADER, $token);
+			$this->server->httpResponse->addHeader(self::PAGINATE_TOTAL_HEADER, (string)$count);
+			$request->setHeader(self::PAGINATE_TOKEN_HEADER, $token);
+		}
+	}
+
+	public function onMethod(RequestInterface $request, ResponseInterface $response) {
+		$url = $this->server->httpRequest->getUrl();
+		if (
+			$request->hasHeader(self::PAGINATE_TOKEN_HEADER) &&
+			$request->hasHeader(self::PAGINATE_OFFSET_HEADER) &&
+			$this->cache->exists($url, $request->getHeader(self::PAGINATE_TOKEN_HEADER))
+		) {
+			$token = $request->getHeader(self::PAGINATE_TOKEN_HEADER);
+			$offset = (int)$request->getHeader(self::PAGINATE_OFFSET_HEADER);
+			$count = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize;
+
+			$items = $this->cache->get($url, $token, $offset, $count);
+
+			$response->setStatus(207);
+			$response->addHeader(self::PAGINATE_HEADER, 'true');
+			$response->setHeader('Content-Type', 'application/xml; charset=utf-8');
+			$response->setHeader('Vary', 'Brief,Prefer');
+
+			$prefer = $this->server->getHTTPPrefer();
+			$minimal = $prefer['return'] === 'minimal';
+
+			$data = $this->server->generateMultiStatus($items, $minimal);
+			$response->setBody($data);
+
+			return false;
+		}
+	}
+}

apps/dav/lib/Server.php

@@ -57,6 +57,7 @@
 use OCA\DAV\Files\BrowserErrorPagePlugin;
 use OCA\DAV\Files\FileSearchBackend;
 use OCA\DAV\Files\LazySearchBackend;
+use OCA\DAV\Paginate\PaginatePlugin;
 use OCA\DAV\Profiler\ProfilerPlugin;
 use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin;
 use OCA\DAV\SystemTag\SystemTagPlugin;
@@ -228,6 +229,7 @@ public function __construct(
 			$logger,
 			$eventDispatcher,
 		));
+		$this->server->addPlugin(\OCP\Server::get(PaginatePlugin::class));
 
 		// allow setup of additional plugins
 		$eventDispatcher->dispatch('OCA\DAV\Connector\Sabre::addPlugin', $event);

apps/dav/lib/SystemTag/SystemTagList.php

@@ -19,18 +19,20 @@
  */
 class SystemTagList implements Element {
 	public const NS_NEXTCLOUD = 'http://nextcloud.org/ns';
+	private array $canAssignTagMap = [];
 
 	/**
 	 * @param ISystemTag[] $tags
 	 */
 	public function __construct(
 		private array $tags,
-		private ISystemTagManager $tagManager,
-		private ?IUser $user,
+		ISystemTagManager $tagManager,
+		?IUser $user,
 	) {
 		$this->tags = $tags;
-		$this->tagManager = $tagManager;
-		$this->user = $user;
+		foreach ($this->tags as $tag) {
+			$this->canAssignTagMap[$tag->getId()] = $tagManager->canUserAssignTag($tag, $user);
+		}
 	}
 
 	/**
@@ -48,7 +50,7 @@ public function xmlSerialize(Writer $writer): void {
 		foreach ($this->tags as $tag) {
 			$writer->startElement('{' . self::NS_NEXTCLOUD . '}system-tag');
 			$writer->writeAttributes([
-				SystemTagPlugin::CANASSIGN_PROPERTYNAME => $this->tagManager->canUserAssignTag($tag, $this->user) ? 'true' : 'false',
+				SystemTagPlugin::CANASSIGN_PROPERTYNAME => $this->canAssignTagMap[$tag->getId()] ? 'true' : 'false',
 				SystemTagPlugin::ID_PROPERTYNAME => $tag->getId(),
 				SystemTagPlugin::USERASSIGNABLE_PROPERTYNAME => $tag->isUserAssignable() ? 'true' : 'false',
 				SystemTagPlugin::USERVISIBLE_PROPERTYNAME => $tag->isUserVisible() ? 'true' : 'false',