feat(fs): speedup s3 delete tree 8-10/s => 100-120/s

vatesfr · Jan 20, 2022 · b6839d7 · b6839d7
1 parent 38aacdb
commit b6839d7
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 3 deletions.
diff --git a/@xen-orchestra/backups/RemoteAdapter.js b/@xen-orchestra/backups/RemoteAdapter.js
@@ -277,6 +277,12 @@ class RemoteAdapter {
       delta !== undefined && this.deleteDeltaVmBackups(delta),
       full !== undefined && this.deleteFullVmBackups(full),
     ])
+
+    const dirs = new Set(files.map(file => dirname(file)))
+    for (const dir of dirs) {
+      // don't merge in main process, unused VHDs will be merged in the next backup run
+      await this.cleanVm(dir, { remove: true, onLog: warn })
+    }
   }
 
   #getCompressionType() {

diff --git a/@xen-orchestra/fs/src/s3.js b/@xen-orchestra/fs/src/s3.js
@@ -177,7 +177,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
       Delimiter: '/', // will only return path until delimiters
     })
 
-    if (result.isTruncated) {
+    if (result.IsTruncated) {
       const error = new Error('more than 1000 objects, unsupported in this implementation')
       error.dir = dir
       throw error
@@ -267,7 +267,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
         Prefix: this._dir + path + '/',
         ContinuationToken: NextContinuationToken,
       })
-      NextContinuationToken = result.isTruncated ? result.NextContinuationToken : undefined
+      NextContinuationToken = result.IsTruncated ? result.NextContinuationToken : undefined
       for (const { Key } of result.Contents) {
         // _unlink will add the prefix, but Key contains everything
         // also we don't need to check if we delete a directory, since the list only return files

diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -21,7 +21,8 @@
 
 - [Backup] Detect and clear orphan merge states, fix `ENOENT` errors (PR [#6087](https://github.com/vatesfr/xen-orchestra/pull/6087))
 - [Backup] Ensure merges are also executed after backup on S3, maintaining the size of the VHD chain under control [Forum#45743](https://xcp-ng.org/forum/post/45743) (PR [#6095](https://github.com/vatesfr/xen-orchestra/pull/6095))
-
+- [Backup] Delete backups immediately instead of waiting for the next backup (PR [#6081](https://github.com/vatesfr/xen-orchestra/pull/6081))
+- [Backup] Delete S3 backup completly , even if there are more than 1000 files (PR [#6103](https://github.com/vatesfr/xen-orchestra/pull/6103))
 ### Packages to release
 
 > Packages will be released in the order they are here, therefore, they should