From a88d4e0e550c10e4d13d4e577925c151370023d9 Mon Sep 17 00:00:00 2001 From: Git'Fellow <12234510+solracsf@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:41:36 +0100 Subject: [PATCH] Max file size for metadata generation Signed-off-by: Git'Fellow <12234510+solracsf@users.noreply.github.com> enh(metadata): Introduce a memory limit for metadata generation Signed-off-by: Git'Fellow <12234510+solracsf@users.noreply.github.com> From Bytes to MiB Signed-off-by: Git'Fellow <12234510+solracsf@users.noreply.github.com> Fix psalm Signed-off-by: Git'Fellow <12234510+solracsf@users.noreply.github.com> Rename var to max_filesize Signed-off-by: Git'Fellow <12234510+solracsf@users.noreply.github.com> Co-Authored-By: Louis --- config/config.sample.php | 9 +++++++++ core/BackgroundJobs/GenerateMetadataJob.php | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/config/config.sample.php b/config/config.sample.php index ae37c3aec9218..8b20ad08284f3 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -1324,6 +1324,15 @@ 'OC\Preview\XBitmap', ], +/** + * Maximum file size for metadata generation. + * If a file exceeds this size, metadata generation will be skipped. + * Note: memory equivalent to this size will be used for metadata generation. + * + * Default: 256 megabytes. + */ +'metadata_max_filesize' => 256, + /** * LDAP * diff --git a/core/BackgroundJobs/GenerateMetadataJob.php b/core/BackgroundJobs/GenerateMetadataJob.php index 65a687649b833..9377a24bce911 100644 --- a/core/BackgroundJobs/GenerateMetadataJob.php +++ b/core/BackgroundJobs/GenerateMetadataJob.php @@ -37,6 +37,9 @@ use Psr\Log\LoggerInterface; class GenerateMetadataJob extends TimedJob { + // Default file size limit for metadata generation (MBytes). + protected const DEFAULT_MAX_FILESIZE = 256; + public function __construct( ITimeFactory $time, private IConfig $config, @@ -100,6 +103,15 @@ private function scanFolder(Folder $folder): void { continue; } + // Don't generate metadata for files bigger than configured metadata_max_filesize + // Files are loaded in memory so very big files can lead to an OOM on the server + $nodeSize = $node->getSize(); + $nodeLimit = $this->config->getSystemValueInt('metadata_max_filesize', self::DEFAULT_MAX_FILESIZE); + if ($nodeSize > $nodeLimit * 1000000) { + $this->logger->debug("Skipping generating metadata for fileid " . $node->getId() . " as its size exceeds configured 'metadata_max_filesize'."); + continue; + } + try { $this->filesMetadataManager->getMetadata($node->getId(), false); } catch (FilesMetadataNotFoundException) {