Change IMetaProxy API to return an ImmutableList of AbstractBackupPat…

…hs when fetching incrementals. The iterators are always fully materialized. Also, remove the now-redundant method from BackupRestoreUtil that merely wrapped the MetaProxy call.
Netflix · May 4, 2024 · d51863c · d51863c
1 parent b80bfb3
commit d51863c
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 58 deletions.
diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java
@@ -21,7 +21,6 @@
 import com.netflix.priam.backupv2.IMetaProxy;
 import com.netflix.priam.utils.DateUtil;
 import java.nio.file.Path;
-import java.time.Instant;
 import java.util.*;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -83,19 +82,6 @@ public static List<AbstractBackupPath> getMostRecentSnapshotPaths(
         return snapshotPaths;
     }
 
-    public static List<AbstractBackupPath> getIncrementalPaths(
-            AbstractBackupPath latestValidMetaFile,
-            DateUtil.DateRange dateRange,
-            IMetaProxy metaProxy) {
-        Instant snapshotTime;
-        snapshotTime = latestValidMetaFile.getLastModified();
-        DateUtil.DateRange incrementalDateRange =
-                new DateUtil.DateRange(snapshotTime, dateRange.getEndTime());
-        List<AbstractBackupPath> incrementalPaths = new ArrayList<>();
-        metaProxy.getIncrementals(incrementalDateRange).forEachRemaining(incrementalPaths::add);
-        return incrementalPaths;
-    }
-
     public static Map<String, List<String>> getFilter(String inputFilter)
             throws IllegalArgumentException {
         if (StringUtils.isEmpty(inputFilter)) return null;

diff --git a/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java b/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java
@@ -17,12 +17,12 @@
 
 package com.netflix.priam.backupv2;
 
+import com.google.common.collect.ImmutableList;
 import com.netflix.priam.backup.AbstractBackupPath;
 import com.netflix.priam.backup.BackupRestoreException;
 import com.netflix.priam.backup.BackupVerificationResult;
 import com.netflix.priam.utils.DateUtil;
 import java.nio.file.Path;
-import java.util.Iterator;
 import java.util.List;
 
 /** Proxy to do management tasks for meta files. Created by aagrawal on 12/18/18. */
@@ -78,7 +78,7 @@ public interface IMetaProxy {
      * @param dateRange the time period to scan in the remote file system for incremental files.
      * @return iterator containing the list of path on the remote file system satisfying criteria.
      */
-    Iterator<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange);
+    ImmutableList<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange);
 
     /**
      * Validate that all the files mentioned in the meta file actually exists on remote file system.

diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java
@@ -17,6 +17,7 @@
 
 package com.netflix.priam.backupv2;
 
+import com.google.common.collect.ImmutableList;
 import com.netflix.priam.backup.AbstractBackupPath;
 import com.netflix.priam.backup.BackupRestoreException;
 import com.netflix.priam.backup.BackupVerificationResult;
@@ -31,8 +32,6 @@
 import java.util.*;
 import javax.inject.Inject;
 import javax.inject.Provider;
-import org.apache.commons.collections4.iterators.FilterIterator;
-import org.apache.commons.collections4.iterators.TransformIterator;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.filefilter.FileFilterUtils;
 import org.apache.commons.io.filefilter.IOFileFilter;
@@ -84,7 +83,7 @@ private String getMatch(
     }
 
     @Override
-    public Iterator<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange) {
+    public ImmutableList<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange) {
         String incrementalPrefix = getMatch(dateRange, AbstractBackupPath.BackupFileType.SST_V2);
         String marker =
                 getMatch(
@@ -96,18 +95,15 @@ public Iterator<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange
                 marker,
                 dateRange);
         Iterator<String> iterator = fs.listFileSystem(incrementalPrefix, null, marker);
-        Iterator<AbstractBackupPath> transformIterator =
-                new TransformIterator<>(
-                        iterator,
-                        s -> {
-                            AbstractBackupPath path = abstractBackupPathProvider.get();
-                            path.parseRemote(s);
-                            return path;
-                        });
-
-        return new FilterIterator<>(
-                transformIterator,
-                abstractBackupPath -> dateRange.contains(abstractBackupPath.getLastModified()));
+        ImmutableList.Builder<AbstractBackupPath> results = ImmutableList.builder();
+        while (iterator.hasNext()) {
+            AbstractBackupPath path = abstractBackupPathProvider.get();
+            path.parseRemote(iterator.next());
+            if (dateRange.contains(path.getLastModified())) {
+                results.add(path);
+            }
+        }
+        return results.build();
     }
 
     @Override

diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java
@@ -153,25 +153,19 @@ public Response validateV2SnapshotByDate(
     @Path("/list/{daterange}")
     public Response list(@PathParam("daterange") String daterange) throws Exception {
         DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange);
-        // Find latest valid meta file.
-        Optional<AbstractBackupPath> latestValidMetaFile =
+        Optional<AbstractBackupPath> metaFile =
                 BackupRestoreUtil.getLatestValidMetaPath(metaProxy, dateRange);
-        if (!latestValidMetaFile.isPresent()) {
+        if (!metaFile.isPresent()) {
             return Response.ok("No valid meta found!").build();
         }
-        List<AbstractBackupPath> allFiles =
+        List<AbstractBackupPath> files =
                 BackupRestoreUtil.getMostRecentSnapshotPaths(
-                        latestValidMetaFile.get(), metaProxy, pathProvider);
-        allFiles.addAll(
-                BackupRestoreUtil.getIncrementalPaths(
-                        latestValidMetaFile.get(), dateRange, metaProxy));
-
-        return Response.ok(
-                        GsonJsonSerializer.getGson()
-                                .toJson(
-                                        allFiles.stream()
-                                                .map(AbstractBackupPath::getRemotePath)
-                                                .collect(Collectors.toList())))
-                .build();
+                        metaFile.get(), metaProxy, pathProvider);
+        DateUtil.DateRange incrementalDateRange =
+                new DateRange(metaFile.get().getLastModified(), dateRange.getEndTime());
+        files.addAll(metaProxy.getIncrementals(incrementalDateRange));
+        List<String> remotePaths =
+                files.stream().map(AbstractBackupPath::getRemotePath).collect(Collectors.toList());
+        return Response.ok(GsonJsonSerializer.getGson().toJson(remotePaths)).build();
     }
 }
diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java
@@ -220,9 +220,11 @@ public void restore(DateUtil.DateRange dateRange) throws Exception {
                     BackupRestoreUtil.getMostRecentSnapshotPaths(
                             latestValidMetaFile.get(), metaProxy, pathProvider);
             if (!config.skipIncrementalRestore()) {
-                allFiles.addAll(
-                        BackupRestoreUtil.getIncrementalPaths(
-                                latestValidMetaFile.get(), dateRange, metaProxy));
+                DateUtil.DateRange incrementalDateRange =
+                        new DateUtil.DateRange(
+                                latestValidMetaFile.get().getLastModified(),
+                                dateRange.getEndTime());
+                allFiles.addAll(metaProxy.getIncrementals(incrementalDateRange));
             }
 
             // Download snapshot which is listed in the meta file.

diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java
@@ -17,6 +17,7 @@
 
 package com.netflix.priam.backupv2;
 
+import com.google.common.truth.Truth;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.netflix.priam.backup.AbstractBackupPath;
@@ -32,7 +33,6 @@
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
 import java.util.stream.Collectors;
 import javax.inject.Provider;
@@ -131,13 +131,7 @@ public void testGetSSTFilesFromMeta() throws Exception {
     @Test
     public void testGetIncrementalFiles() throws Exception {
         DateUtil.DateRange dateRange = new DateUtil.DateRange("202812071820,20281229");
-        Iterator<AbstractBackupPath> incrementals = metaProxy.getIncrementals(dateRange);
-        int i = 0;
-        while (incrementals.hasNext()) {
-            System.out.println(incrementals.next());
-            i++;
-        }
-        Assert.assertEquals(3, i);
+        Truth.assertThat(metaProxy.getIncrementals(dateRange)).hasSize(3);
     }
 
     @Test