Use S3 batch deletes for Delta Lake vacuum procedure

trinodb · Nov 19, 2022 · 2332ed7 · 2332ed7
1 parent 916ece2
commit 2332ed7
Showing 1 changed file with 14 additions and 2 deletions.
diff --git a/...n/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/procedure/VacuumProcedure.java b/...n/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/procedure/VacuumProcedure.java
@@ -46,6 +46,7 @@
 import java.io.IOException;
 import java.lang.invoke.MethodHandle;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -68,6 +69,7 @@ public class VacuumProcedure
         implements Provider<Procedure>
 {
     private static final Logger log = Logger.get(VacuumProcedure.class);
+    private static final int DELETE_BATCH_SIZE = 1000;
 
     private static final MethodHandle VACUUM;
 
@@ -209,6 +211,7 @@ private void doVacuum(
         long retainedUnknownFiles = 0;
         long removedFiles = 0;
 
+        List<String> filesToDelete = new ArrayList<>();
         FileIterator listing = fileSystem.listFiles(tableLocation.toString());
         while (listing.hasNext()) {
             FileEntry entry = listing.next();
@@ -254,8 +257,17 @@ private void doVacuum(
                     path,
                     modificationTime,
                     modificationInstant);
-            fileSystem.deleteFile(path);
-            removedFiles++;
+            filesToDelete.add(path);
+            if (filesToDelete.size() == DELETE_BATCH_SIZE) {
+                fileSystem.deleteFiles(filesToDelete);
+                removedFiles += filesToDelete.size();
+                filesToDelete.clear();
+            }
+        }
+
+        if (!filesToDelete.isEmpty()) {
+            fileSystem.deleteFiles(filesToDelete);
+            removedFiles += filesToDelete.size();
         }
 
         log.info(