From 57f7f0278e7830bdb6990970ce9679810e379d9e Mon Sep 17 00:00:00 2001 From: Anna Larch Date: Wed, 8 May 2024 20:38:51 +0200 Subject: [PATCH] fix(caldav): automatically delete outdated scheduling objects Signed-off-by: Anna Larch --- .../composer/composer/autoload_classmap.php | 2 + .../dav/composer/composer/autoload_static.php | 2 + .../DeleteOutdatedSchedulingObjects.php | 35 ++++++++++++++ apps/dav/lib/CalDAV/CalDavBackend.php | 38 +++++++++++++++ .../lib/Migration/DeleteSchedulingObjects.php | 38 +++++++++++++++ .../Version1004Date20170825134824.php | 1 + .../composer/composer/autoload_classmap.php | 1 + .../composer/composer/autoload_static.php | 1 + apps/settings/lib/AppInfo/Application.php | 2 + .../lib/SetupChecks/SchedulingTableSize.php | 48 +++++++++++++++++++ core/Application.php | 6 +++ lib/private/Repair.php | 2 + 12 files changed, 176 insertions(+) create mode 100644 apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php create mode 100644 apps/dav/lib/Migration/DeleteSchedulingObjects.php create mode 100644 apps/settings/lib/SetupChecks/SchedulingTableSize.php diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php index 0fbff74369dc3..f3cfdf49c4400 100644 --- a/apps/dav/composer/composer/autoload_classmap.php +++ b/apps/dav/composer/composer/autoload_classmap.php @@ -16,6 +16,7 @@ 'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => $baseDir . '/../lib/BackgroundJob/CalendarRetentionJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => $baseDir . '/../lib/BackgroundJob/CleanupDirectLinksJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php', + 'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => $baseDir . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php', 'OCA\\DAV\\BackgroundJob\\EventReminderJob' => $baseDir . '/../lib/BackgroundJob/EventReminderJob.php', 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\OutOfOfficeEventDispatcherJob' => $baseDir . '/../lib/BackgroundJob/OutOfOfficeEventDispatcherJob.php', @@ -281,6 +282,7 @@ 'OCA\\DAV\\Migration\\BuildSocialSearchIndexBackgroundJob' => $baseDir . '/../lib/Migration/BuildSocialSearchIndexBackgroundJob.php', 'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => $baseDir . '/../lib/Migration/CalDAVRemoveEmptyValue.php', 'OCA\\DAV\\Migration\\ChunkCleanup' => $baseDir . '/../lib/Migration/ChunkCleanup.php', + 'OCA\\DAV\\Migration\\DeleteSchedulingObjects' => $baseDir . '/../lib/Migration/DeleteSchedulingObjects.php', 'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => $baseDir . '/../lib/Migration/FixBirthdayCalendarComponent.php', 'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => $baseDir . '/../lib/Migration/RefreshWebcalJobRegistrar.php', 'OCA\\DAV\\Migration\\RegenerateBirthdayCalendars' => $baseDir . '/../lib/Migration/RegenerateBirthdayCalendars.php', diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php index a800e77364235..b8a3d0d5964b6 100644 --- a/apps/dav/composer/composer/autoload_static.php +++ b/apps/dav/composer/composer/autoload_static.php @@ -31,6 +31,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CalendarRetentionJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupDirectLinksJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php', + 'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => __DIR__ . '/..' . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php', 'OCA\\DAV\\BackgroundJob\\EventReminderJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/EventReminderJob.php', 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\OutOfOfficeEventDispatcherJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/OutOfOfficeEventDispatcherJob.php', @@ -296,6 +297,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\Migration\\BuildSocialSearchIndexBackgroundJob' => __DIR__ . '/..' . '/../lib/Migration/BuildSocialSearchIndexBackgroundJob.php', 'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => __DIR__ . '/..' . '/../lib/Migration/CalDAVRemoveEmptyValue.php', 'OCA\\DAV\\Migration\\ChunkCleanup' => __DIR__ . '/..' . '/../lib/Migration/ChunkCleanup.php', + 'OCA\\DAV\\Migration\\DeleteSchedulingObjects' => __DIR__ . '/..' . '/../lib/Migration/DeleteSchedulingObjects.php', 'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => __DIR__ . '/..' . '/../lib/Migration/FixBirthdayCalendarComponent.php', 'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => __DIR__ . '/..' . '/../lib/Migration/RefreshWebcalJobRegistrar.php', 'OCA\\DAV\\Migration\\RegenerateBirthdayCalendars' => __DIR__ . '/..' . '/../lib/Migration/RegenerateBirthdayCalendars.php', diff --git a/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php b/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php new file mode 100644 index 0000000000000..fa53a8be4f02a --- /dev/null +++ b/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php @@ -0,0 +1,35 @@ +setInterval(23 * 60 * 60); + $this->setTimeSensitivity(self::TIME_INSENSITIVE); + } + + /** + * @param array $argument + */ + protected function run($argument): void { + $time = $this->time->getTime() - (60 * 60); + $this->calDavBackend->deleteOutdatedSchedulingObjects($time, 50000); + $this->logger->info("Removed outdated scheduling objects"); + } +} diff --git a/apps/dav/lib/CalDAV/CalDavBackend.php b/apps/dav/lib/CalDAV/CalDavBackend.php index acea7b4aeab0b..cc97206493cbc 100644 --- a/apps/dav/lib/CalDAV/CalDavBackend.php +++ b/apps/dav/lib/CalDAV/CalDavBackend.php @@ -2729,6 +2729,44 @@ public function deleteSchedulingObject($principalUri, $objectUri) { ->executeStatement(); } + /** + * Deletes all scheduling objects last modified before $modifiedBefore from the inbox collection. + * + * @param int $modifiedBefore + * @param int $limit + * @return void + */ + public function deleteOutdatedSchedulingObjects(int $modifiedBefore, int $limit): void { + $query = $this->db->getQueryBuilder(); + $query->select('id') + ->from('schedulingobjects') + ->where($query->expr()->lt('lastmodified', $query->createNamedParameter($modifiedBefore))) + ->setMaxResults($limit); + $result = $query->executeQuery(); + $count = $result->rowCount(); + if($count === 0) { + return; + } + $ids = array_map(static function (array $id) { + return (int)$id[0]; + }, $result->fetchAll(\PDO::FETCH_NUM)); + $result->closeCursor(); + + $numDeleted = 0; + $deleteQuery = $this->db->getQueryBuilder(); + $deleteQuery->delete('schedulingobjects') + ->where($deleteQuery->expr()->in('id', $deleteQuery->createParameter('ids'), IQueryBuilder::PARAM_INT_ARRAY)); + foreach(array_chunk($ids, 1000) as $chunk) { + $deleteQuery->setParameter('ids', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $numDeleted += $deleteQuery->executeStatement(); + } + + if($numDeleted === $limit) { + $this->logger->info("Deleted $limit scheduling objects, continuing with next batch"); + $this->deleteOutdatedSchedulingObjects($modifiedBefore, $limit); + } + } + /** * Creates a new scheduling object. This should land in a users' inbox. * diff --git a/apps/dav/lib/Migration/DeleteSchedulingObjects.php b/apps/dav/lib/Migration/DeleteSchedulingObjects.php new file mode 100644 index 0000000000000..3919236788bb3 --- /dev/null +++ b/apps/dav/lib/Migration/DeleteSchedulingObjects.php @@ -0,0 +1,38 @@ +info('Cleaning up old scheduling events'); + $time = $this->time->getTime() - (60 * 60); + $this->calDavBackend->deleteOutdatedSchedulingObjects($time, 50000); + if (!$this->jobList->has(DeleteOutdatedSchedulingObjects::class, null)) { + $output->info('Adding background job to delete old scheduling objects'); + $this->jobList->add(DeleteOutdatedSchedulingObjects::class, null); + } + } +} diff --git a/apps/dav/lib/Migration/Version1004Date20170825134824.php b/apps/dav/lib/Migration/Version1004Date20170825134824.php index a7cbaa78ef23e..7321bba62ff92 100644 --- a/apps/dav/lib/Migration/Version1004Date20170825134824.php +++ b/apps/dav/lib/Migration/Version1004Date20170825134824.php @@ -383,6 +383,7 @@ public function changeSchema(IOutput $output, \Closure $schemaClosure, array $op ]); $table->setPrimaryKey(['id']); $table->addIndex(['principaluri'], 'schedulobj_principuri_index'); + $table->addIndex(['lastmodified'], 'schedulobj_lastmodified_idx'); } if (!$schema->hasTable('cards_properties')) { diff --git a/apps/settings/composer/composer/autoload_classmap.php b/apps/settings/composer/composer/autoload_classmap.php index ef81604ca0e9f..685c8a1f1219a 100644 --- a/apps/settings/composer/composer/autoload_classmap.php +++ b/apps/settings/composer/composer/autoload_classmap.php @@ -110,6 +110,7 @@ 'OCA\\Settings\\SetupChecks\\PushService' => $baseDir . '/../lib/SetupChecks/PushService.php', 'OCA\\Settings\\SetupChecks\\RandomnessSecure' => $baseDir . '/../lib/SetupChecks/RandomnessSecure.php', 'OCA\\Settings\\SetupChecks\\ReadOnlyConfig' => $baseDir . '/../lib/SetupChecks/ReadOnlyConfig.php', + 'OCA\\Settings\\SetupChecks\\SchedulingTableSize' => $baseDir . '/../lib/SetupChecks/SchedulingTableSize.php', 'OCA\\Settings\\SetupChecks\\SupportedDatabase' => $baseDir . '/../lib/SetupChecks/SupportedDatabase.php', 'OCA\\Settings\\SetupChecks\\SystemIs64bit' => $baseDir . '/../lib/SetupChecks/SystemIs64bit.php', 'OCA\\Settings\\SetupChecks\\TempSpaceAvailable' => $baseDir . '/../lib/SetupChecks/TempSpaceAvailable.php', diff --git a/apps/settings/composer/composer/autoload_static.php b/apps/settings/composer/composer/autoload_static.php index fe23224d21493..65556e9d93e7f 100644 --- a/apps/settings/composer/composer/autoload_static.php +++ b/apps/settings/composer/composer/autoload_static.php @@ -125,6 +125,7 @@ class ComposerStaticInitSettings 'OCA\\Settings\\SetupChecks\\PushService' => __DIR__ . '/..' . '/../lib/SetupChecks/PushService.php', 'OCA\\Settings\\SetupChecks\\RandomnessSecure' => __DIR__ . '/..' . '/../lib/SetupChecks/RandomnessSecure.php', 'OCA\\Settings\\SetupChecks\\ReadOnlyConfig' => __DIR__ . '/..' . '/../lib/SetupChecks/ReadOnlyConfig.php', + 'OCA\\Settings\\SetupChecks\\SchedulingTableSize' => __DIR__ . '/..' . '/../lib/SetupChecks/SchedulingTableSize.php', 'OCA\\Settings\\SetupChecks\\SupportedDatabase' => __DIR__ . '/..' . '/../lib/SetupChecks/SupportedDatabase.php', 'OCA\\Settings\\SetupChecks\\SystemIs64bit' => __DIR__ . '/..' . '/../lib/SetupChecks/SystemIs64bit.php', 'OCA\\Settings\\SetupChecks\\TempSpaceAvailable' => __DIR__ . '/..' . '/../lib/SetupChecks/TempSpaceAvailable.php', diff --git a/apps/settings/lib/AppInfo/Application.php b/apps/settings/lib/AppInfo/Application.php index 3e04250bca78c..1e435b91f0d33 100644 --- a/apps/settings/lib/AppInfo/Application.php +++ b/apps/settings/lib/AppInfo/Application.php @@ -82,6 +82,7 @@ use OCA\Settings\SetupChecks\PushService; use OCA\Settings\SetupChecks\RandomnessSecure; use OCA\Settings\SetupChecks\ReadOnlyConfig; +use OCA\Settings\SetupChecks\SchedulingTableSize; use OCA\Settings\SetupChecks\SupportedDatabase; use OCA\Settings\SetupChecks\SystemIs64bit; use OCA\Settings\SetupChecks\TempSpaceAvailable; @@ -204,6 +205,7 @@ public function register(IRegistrationContext $context): void { $context->registerSetupCheck(PhpOutputBuffering::class); $context->registerSetupCheck(RandomnessSecure::class); $context->registerSetupCheck(ReadOnlyConfig::class); + $context->registerSetupCheck(SchedulingTableSize::class); $context->registerSetupCheck(SupportedDatabase::class); $context->registerSetupCheck(SystemIs64bit::class); $context->registerSetupCheck(TempSpaceAvailable::class); diff --git a/apps/settings/lib/SetupChecks/SchedulingTableSize.php b/apps/settings/lib/SetupChecks/SchedulingTableSize.php new file mode 100644 index 0000000000000..cf629f5b12c9c --- /dev/null +++ b/apps/settings/lib/SetupChecks/SchedulingTableSize.php @@ -0,0 +1,48 @@ +l10n->t('Scheduling objects table size'); + } + + public function getCategory(): string { + return 'database'; + } + + public function run(): SetupResult { + $qb = $this->connection->getQueryBuilder(); + $qb->select($qb->func()->count('id')) + ->from('schedulingobjects'); + $query = $qb->executeQuery(); + $count = $query->fetchOne(); + $query->closeCursor(); + + if ($count > 500000) { + return SetupResult::warning( + $this->l10n->t('You have more than 500 000 rows in the scheduling objects table. Please run the expensive repair jobs via occ maintenance:repair --include-expensive') + ); + } + return SetupResult::success( + $this->l10n->t('Scheduling objects table size is within acceptable range.') + ); + } +} diff --git a/core/Application.php b/core/Application.php index 2ad8b9f2a3014..89952d89520dd 100644 --- a/core/Application.php +++ b/core/Application.php @@ -207,6 +207,12 @@ public function __construct() { ['principaluri'] ); + $event->addMissingIndex( + 'schedulingobjects', + 'schedulobj_lastmodified_idx', + ['lastmodified'] + ); + $event->addMissingIndex( 'properties', 'properties_path_index', diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 21caed3e39fbd..6593e092914bf 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -80,6 +80,7 @@ use OC\Repair\RepairInvalidShares; use OC\Repair\RepairMimeTypes; use OC\Template\JSCombiner; +use OCA\DAV\Migration\DeleteSchedulingObjects; use OCP\AppFramework\QueryException; use OCP\AppFramework\Utility\ITimeFactory; use OCP\Collaboration\Resources\IManager; @@ -226,6 +227,7 @@ public static function getExpensiveRepairSteps() { return [ new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()), \OC::$server->get(ValidatePhoneNumber::class), + \OC::$server->get(DeleteSchedulingObjects::class), ]; }