Skip to content

Commit

Permalink
Return missing files from snapshot verification continuing to margina…
Browse files Browse the repository at this point in the history 
…lize BackupVerificationResult. Return all results valid or otherwise.
  • Loading branch information
@mattl-netflix
mattl-netflix committed Apr 2, 2023
1 parent 373069a commit 444d20e
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 43 deletions.
29 changes: 16 additions & 13 deletions priam/src/main/java/com/netflix/priam/backup/BackupVerification.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
*/
package com.netflix.priam.backup;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.netflix.priam.backupv2.IMetaProxy;
import com.netflix.priam.utils.DateUtil;
import com.netflix.priam.utils.DateUtil.DateRange;
Expand Down Expand Up @@ -75,9 +77,9 @@ public Optional<BackupVerificationResult> verifyLatestBackup(
for (BackupMetadata backupMetadata :
backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) {
if (backupMetadata.getLastValidated() == null || force) {
Optional<BackupVerificationResult> result = verifyBackup(metaProxy, backupMetadata);
if (result.isPresent()) {
return result;
BackupVerificationResult result = verifyBackup(metaProxy, backupMetadata);
if (result.valid) {
return Optional.of(result);
}
} else {
updateLatestResult(backupMetadata);
Expand All @@ -88,26 +90,28 @@ public Optional<BackupVerificationResult> verifyLatestBackup(
return Optional.empty();
}

public List<BackupMetadata> verifyBackupsInRange(
public ImmutableMap<BackupMetadata, ImmutableSet<String>> findMissingBackupFilesInRange(
BackupVersion backupVersion, DateRange dateRange) throws IllegalArgumentException {
IMetaProxy metaProxy = getMetaProxy(backupVersion);
List<BackupMetadata> results = new ArrayList<>();
ImmutableMap.Builder<BackupMetadata, ImmutableSet<String>> mapBuilder =
ImmutableMap.builder();
for (BackupMetadata backupMetadata :
backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) {
if (backupMetadata.getLastValidated() != null
|| verifyBackup(metaProxy, backupMetadata).isPresent()) {
results.add(backupMetadata);
}
List<String> missingFiles =
backupMetadata.getLastValidated() == null
? verifyBackup(metaProxy, backupMetadata).filesInMetaOnly
: new ArrayList<>();
mapBuilder.put(backupMetadata, ImmutableSet.copyOf(missingFiles));
}
return results;
return mapBuilder.build();
}

/** returns the latest valid backup verification result if we have found one within the SLO * */
public Optional<Instant> getLatestVerfifiedBackupTime() {
return latestResult == null ? Optional.empty() : Optional.of(latestResult.snapshotInstant);
}

private Optional<BackupVerificationResult> verifyBackup(
private BackupVerificationResult verifyBackup(
IMetaProxy metaProxy, BackupMetadata latestBackupMetaData) {
Path metadataLocation = Paths.get(latestBackupMetaData.getSnapshotLocation());
metadataLocation = metadataLocation.subpath(1, metadataLocation.getNameCount());
Expand All @@ -119,9 +123,8 @@ private Optional<BackupVerificationResult> verifyBackup(
Date now = new Date(DateUtil.getInstant().toEpochMilli());
latestBackupMetaData.setLastValidated(now);
backupStatusMgr.update(latestBackupMetaData);
return Optional.of(result);
}
return Optional.empty();
return result;
}

private void updateLatestResult(BackupMetadata backupMetadata) {
Expand Down
25 changes: 17 additions & 8 deletions priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@
import com.netflix.priam.utils.DateUtil.DateRange;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.slf4j.Logger;
Expand Down Expand Up @@ -84,21 +86,28 @@ public void execute() throws Exception {
Instant slo =
now.minus(backupRestoreConfig.getBackupVerificationSLOInHours(), ChronoUnit.HOURS);
DateRange dateRange = new DateRange(slo, now);
List<BackupMetadata> verifiedBackups =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_META_SERVICE, dateRange);
Set<BackupMetadata> verifiedBackups =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE, dateRange)
.entrySet()
.stream()
.filter(entry -> entry.getValue().isEmpty())
.map(Map.Entry::getKey)
.collect(Collectors.toSet());

verifiedBackups
.stream()
.filter(result -> result.getLastValidated().toInstant().isAfter(now))
.filter(metadata -> metadata.getLastValidated().toInstant().isAfter(now))
.forEach(
result -> {
metadata -> {
logger.info(
"Sending {} message for backup: {}",
AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED,
result.getSnapshotLocation());
metadata.getSnapshotLocation());
backupNotificationMgr.notify(
result.getSnapshotLocation(), result.getStart().toInstant());
metadata.getSnapshotLocation(),
metadata.getStart().toInstant());
});

if (verifiedBackups.isEmpty()) {
Expand Down
59 changes: 38 additions & 21 deletions priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package com.netflix.priam.backup;

import com.google.common.truth.Truth;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.netflix.priam.backup.AbstractBackupPath.BackupFileType;
Expand All @@ -31,6 +32,7 @@
import java.nio.file.Paths;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.AbstractCollection;
import java.util.Date;
import java.util.List;
import java.util.Optional;
Expand Down Expand Up @@ -109,16 +111,23 @@ public void noBackup() throws Exception {

@Test
public void noBackupDateRange() throws Exception {
List<BackupMetadata> backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_BACKUP, new DateRange(Instant.now(), Instant.now()));
Assert.assertFalse(backupVerificationResults.size() > 0);
long foundBackups =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP,
new DateRange(Instant.now(), Instant.now()))
.entrySet()
.size();
Truth.assertThat(foundBackups).isEqualTo(0L);

backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
new DateRange(Instant.now(), Instant.now()));
Assert.assertFalse(backupVerificationResults.size() > 0);
foundBackups =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
new DateRange(Instant.now(), Instant.now()))
.entrySet()
.size();
Truth.assertThat(foundBackups).isEqualTo(0L);
}

private void setUp() throws Exception {
Expand Down Expand Up @@ -184,12 +193,16 @@ public void verifyBackupVersion1() throws Exception {
public void verifyBackupVersion1DateRange() throws Exception {
setUp();
// Verify for backup version 1.0
List<BackupMetadata> backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_BACKUP,
new DateRange(backupDate + "," + backupDateEnd));
Assert.assertTrue(!backupVerificationResults.isEmpty());
Assert.assertTrue(backupVerificationResults.size() == numFakeBackups);
long missingFilesCount =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP,
new DateRange(backupDate + "," + backupDateEnd))
.values()
.stream()
.filter(AbstractCollection::isEmpty)
.count();
Truth.assertThat(missingFilesCount).isEqualTo(numFakeBackups);
List<BackupMetadata> backupMetadata =
backupStatusMgr.getLatestBackupMetadata(
BackupVersion.SNAPSHOT_BACKUP,
Expand Down Expand Up @@ -260,12 +273,16 @@ public void verifyBackupVersion2() throws Exception {
public void verifyBackupVersion2DateRange() throws Exception {
setUp();
// Verify for backup version 2.0
List<BackupMetadata> backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
new DateRange(backupDate + "," + backupDateEnd));
Assert.assertTrue(!backupVerificationResults.isEmpty());
Assert.assertTrue(backupVerificationResults.size() == numFakeBackups);
long missingFilesCount =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
new DateRange(backupDate + "," + backupDateEnd))
.values()
.stream()
.filter(AbstractCollection::isEmpty)
.count();
Truth.assertThat(missingFilesCount).isEqualTo(numFakeBackups);
List<BackupMetadata> backupMetadata =
backupStatusMgr.getLatestBackupMetadata(
BackupVersion.SNAPSHOT_META_SERVICE,
Expand Down
3 changes: 2 additions & 1 deletion priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,8 @@ public void testRestoreMode(@Mocked InstanceState state) throws Exception {
Truth.assertThat(badVerifications.count()).isEqualTo(0);
new Verifications() {
{
backupVerification.verifyBackupsInRange((BackupVersion) any, (DateRange) any);
backupVerification.findMissingBackupFilesInRange(
(BackupVersion) any, (DateRange) any);
maxTimes = 0;
}

Expand Down

0 comments on commit 444d20e

Please sign in to comment.