From 2ea6713504aa49d1a8cb9a7efaad8706484986c1 Mon Sep 17 00:00:00 2001 From: Git'Fellow <12234510+solracsf@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:41:36 +0100 Subject: [PATCH] fix: Max file size for metadata generation Signed-off-by: Git'Fellow <12234510+solracsf@users.noreply.github.com> Co-Authored-By: Louis Signed-off-by: Richard Steinmetz --- config/config.sample.php | 9 +++++++++ core/BackgroundJobs/GenerateMetadataJob.php | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/config/config.sample.php b/config/config.sample.php index 8b3c9feae45b7..f2d06d9acea92 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -1373,6 +1373,15 @@ 'OC\Preview\XBitmap', ], +/** + * Maximum file size for metadata generation. + * If a file exceeds this size, metadata generation will be skipped. + * Note: memory equivalent to this size will be used for metadata generation. + * + * Default: 256 megabytes. + */ +'metadata_max_filesize' => 256, + /** * LDAP * diff --git a/core/BackgroundJobs/GenerateMetadataJob.php b/core/BackgroundJobs/GenerateMetadataJob.php index 809e139e6ef0f..c40a1b11f069a 100644 --- a/core/BackgroundJobs/GenerateMetadataJob.php +++ b/core/BackgroundJobs/GenerateMetadataJob.php @@ -16,12 +16,17 @@ use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IAppConfig; +use OCP\IConfig; use OCP\IUserManager; use Psr\Log\LoggerInterface; class GenerateMetadataJob extends TimedJob { + // Default file size limit for metadata generation (MBytes). + protected const DEFAULT_MAX_FILESIZE = 256; + public function __construct( ITimeFactory $time, + private IConfig $config, private IAppConfig $appConfig, private IRootFolder $rootFolder, private IUserManager $userManager, @@ -88,6 +93,15 @@ private function scanFolder(Folder $folder): void { continue; } + // Don't generate metadata for files bigger than configured metadata_max_filesize + // Files are loaded in memory so very big files can lead to an OOM on the server + $nodeSize = $node->getSize(); + $nodeLimit = $this->config->getSystemValueInt('metadata_max_filesize', self::DEFAULT_MAX_FILESIZE); + if ($nodeSize > $nodeLimit * 1000000) { + $this->logger->debug("Skipping generating metadata for fileid " . $node->getId() . " as its size exceeds configured 'metadata_max_filesize'."); + continue; + } + try { $this->filesMetadataManager->getMetadata($node->getId(), false); } catch (FilesMetadataNotFoundException) {