Merge pull request #46583 from nextcloud/filecache-sharding-compat

Make filecache queries compatible with sharding

lib/private/Files/Cache/Cache.php

@@ -120,11 +120,11 @@ public function get($file) {
 			// normalize file
 			$file = $this->normalize($file);
 
-			$query->whereStorageId($this->getNumericStorageId())
-				->wherePath($file);
+			$query->wherePath($file);
 		} else { //file id
 			$query->whereFileId($file);
 		}
+		$query->whereStorageId($this->getNumericStorageId());
 
 		$result = $query->execute();
 		$data = $result->fetch();
@@ -199,6 +199,7 @@ public function getFolderContentsById($fileId) {
 			$query = $this->getQueryBuilder();
 			$query->selectFileCache()
 				->whereParent($fileId)
+				->whereStorageId($this->getNumericStorageId())
 				->orderBy('name', 'ASC');
 
 			$metadataQuery = $query->selectMetadata();
@@ -337,6 +338,7 @@ public function update($id, array $data) {
 
 			$query->update('filecache')
 				->whereFileId($id)
+				->whereStorageId($this->getNumericStorageId())
 				->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) {
 					return $query->expr()->orX(
 						$query->expr()->neq($key, $query->createNamedParameter($value)),
@@ -512,6 +514,7 @@ public function remove($file) {
 		if ($entry instanceof ICacheEntry) {
 			$query = $this->getQueryBuilder();
 			$query->delete('filecache')
+				->whereStorageId($this->getNumericStorageId())
 				->whereFileId($entry->getId());
 			$query->execute();
 
@@ -583,6 +586,7 @@ private function removeChildren(ICacheEntry $entry) {
 
 		$query = $this->getQueryBuilder();
 		$query->delete('filecache')
+			->whereStorageId($this->getNumericStorageId())
 			->whereParentInParameter('parentIds');
 
 		// Sorting before chunking allows the db to find the entries close to each
@@ -929,6 +933,7 @@ protected function calculateFolderSizeInner(string $path, $entry = null, bool $i
 			$query = $this->getQueryBuilder();
 			$query->select('size', 'unencrypted_size')
 				->from('filecache')
+				->whereStorageId($this->getNumericStorageId())
 				->whereParent($id);
 			if ($ignoreUnknown) {
 				$query->andWhere($query->expr()->gte('size', $query->createNamedParameter(0)));

lib/private/Files/Cache/Propagator.php

@@ -186,15 +186,15 @@ public function commitBatch() {
 		$query->update('filecache')
 			->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
 			->set('etag', $query->expr()->literal(uniqid()))
-			->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT)))
+			->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
 			->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')));
 
 		$sizeQuery = $this->connection->getQueryBuilder();
 		$sizeQuery->update('filecache')
 			->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size')))
-			->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT)))
-			->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')))
-			->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->expr()->literal(-1, IQueryBuilder::PARAM_INT)));
+			->where($query->expr()->eq('storage', $sizeQuery->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
+			->andWhere($query->expr()->eq('path_hash', $sizeQuery->createParameter('hash')))
+			->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->createNamedParameter(-1, IQueryBuilder::PARAM_INT)));
 
 		foreach ($this->batch as $item) {
 			$query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT);

lib/private/Files/Cache/QuerySearchHelper.php

@@ -110,7 +110,6 @@ protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user):
 		$query
 			->leftJoin('file', 'vcategory_to_object', 'tagmap', $query->expr()->eq('file.fileid', 'tagmap.objid'))
 			->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX(
-				$query->expr()->eq('tagmap.type', 'tag.type'),
 				$query->expr()->eq('tagmap.categoryid', 'tag.id'),
 				$query->expr()->eq('tag.type', $query->createNamedParameter('files')),
 				$query->expr()->eq('tag.uid', $query->createNamedParameter($user->getUID()))

lib/private/Files/Config/UserMountCache.php

@@ -237,7 +237,7 @@ public function getMountsForUser(IUser $user) {
 			$builder = $this->connection->getQueryBuilder();
 			$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'mount_provider_class')
 				->from('mounts', 'm')
-				->where($builder->expr()->eq('user_id', $builder->createPositionalParameter($userUID)));
+				->where($builder->expr()->eq('user_id', $builder->createNamedParameter($userUID)));
 
 			$result = $query->execute();
 			$rows = $result->fetchAll();
@@ -264,7 +264,7 @@ public function getInternalPathForMountInfo(CachedMountInfo $info): string {
 		$builder = $this->connection->getQueryBuilder();
 		$query = $builder->select('path')
 			->from('filecache')
-			->where($builder->expr()->eq('fileid', $builder->createPositionalParameter($info->getRootId())));
+			->where($builder->expr()->eq('fileid', $builder->createNamedParameter($info->getRootId())));
 		return $query->executeQuery()->fetchOne() ?: '';
 	}
 
@@ -278,10 +278,10 @@ public function getMountsForStorageId($numericStorageId, $user = null) {
 		$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class')
 			->from('mounts', 'm')
 			->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid'))
-			->where($builder->expr()->eq('storage_id', $builder->createPositionalParameter($numericStorageId, IQueryBuilder::PARAM_INT)));
+			->where($builder->expr()->eq('storage_id', $builder->createNamedParameter($numericStorageId, IQueryBuilder::PARAM_INT)));
 
 		if ($user) {
-			$query->andWhere($builder->expr()->eq('user_id', $builder->createPositionalParameter($user)));
+			$query->andWhere($builder->expr()->eq('user_id', $builder->createNamedParameter($user)));
 		}
 
 		$result = $query->execute();
@@ -300,7 +300,7 @@ public function getMountsForRootId($rootFileId) {
 		$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class')
 			->from('mounts', 'm')
 			->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid'))
-			->where($builder->expr()->eq('root_id', $builder->createPositionalParameter($rootFileId, IQueryBuilder::PARAM_INT)));
+			->where($builder->expr()->eq('root_id', $builder->createNamedParameter($rootFileId, IQueryBuilder::PARAM_INT)));
 
 		$result = $query->execute();
 		$rows = $result->fetchAll();

tests/lib/Files/Cache/PropagatorTest.php

@@ -122,7 +122,7 @@ public function testBatchedPropagation() {
 
 		foreach ($oldInfos as $i => $oldInfo) {
 			if ($oldInfo->getPath() !== 'foo/baz') {
-				$this->assertNotEquals($oldInfo->getEtag(), $newInfos[$i]->getEtag());
+				$this->assertNotEquals($oldInfo->getEtag(), $newInfos[$i]->getEtag(), "etag for {$oldInfo->getPath()} not updated");
 			}
 		}
 

tests/lib/Files/Config/UserMountCacheTest.php

@@ -7,11 +7,13 @@
 
 namespace Test\Files\Config;
 
+use OC\DB\Exceptions\DbalException;
 use OC\DB\QueryBuilder\Literal;
 use OC\Files\Mount\MountPoint;
 use OC\Files\Storage\Storage;
 use OC\User\Manager;
 use OCP\Cache\CappedMemoryCache;
+use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Diagnostics\IEventLogger;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\Config\ICachedMountInfo;
@@ -335,29 +337,38 @@ private function sortMounts(&$mounts) {
 
 	private function createCacheEntry($internalPath, $storageId, $size = 0) {
 		$internalPath = trim($internalPath, '/');
-		$inserted = $this->connection->insertIfNotExist('*PREFIX*filecache', [
-			'storage' => $storageId,
-			'path' => $internalPath,
-			'path_hash' => md5($internalPath),
-			'parent' => -1,
-			'name' => basename($internalPath),
-			'mimetype' => 0,
-			'mimepart' => 0,
-			'size' => $size,
-			'storage_mtime' => 0,
-			'encrypted' => 0,
-			'unencrypted_size' => 0,
-			'etag' => '',
-			'permissions' => 31
-		], ['storage', 'path_hash']);
-		if ($inserted) {
-			$id = (int)$this->connection->lastInsertId('*PREFIX*filecache');
+		try {
+			$query = $this->connection->getQueryBuilder();
+			$query->insert('filecache')
+				->values([
+					'storage' => $query->createNamedParameter($storageId),
+					'path' => $query->createNamedParameter($internalPath),
+					'path_hash' => $query->createNamedParameter(md5($internalPath)),
+					'parent' => $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT),
+					'name' => $query->createNamedParameter(basename($internalPath)),
+					'mimetype' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
+					'mimepart' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
+					'size' => $query->createNamedParameter($size),
+					'storage_mtime' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
+					'encrypted' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
+					'unencrypted_size' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
+					'etag' => $query->createNamedParameter(''),
+					'permissions' => $query->createNamedParameter(31, IQueryBuilder::PARAM_INT),
+				]);
+			$query->executeStatement();
+			$id = $query->getLastInsertId();
 			$this->fileIds[] = $id;
-		} else {
-			$sql = 'SELECT `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` =?';
-			$query = $this->connection->prepare($sql);
-			$query->execute([$storageId, md5($internalPath)]);
-			return (int)$query->fetchOne();
+		} catch (DbalException $e) {
+			if ($e->getReason() === DbalException::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
+				$query = $this->connection->getQueryBuilder();
+				$query->select('fileid')
+					->from('filecache')
+					->where($query->expr()->eq('storage', $query->createNamedParameter($storageId)))
+					->andWhere($query->expr()->eq('path_hash', $query->createNamedParameter(md5($internalPath))));
+				$id = (int)$query->execute()->fetchColumn();
+			} else {
+				throw $e;
+			}
 		}
 		return $id;
 	}