From 797bd77edf5727d28898fa94f8f5936bddeff217 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 5 Oct 2022 15:02:46 -0700 Subject: [PATCH 001/120] initial changes with noCopyList --- .../hdds/utils/DBCheckpointServlet.java | 2 +- .../ozone/om/OMDBCheckpointServlet.java | 138 ++++++++++++++++++ 2 files changed, 139 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 03f2fdca390..a6a07beb6fc 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -238,7 +238,7 @@ public static void writeDBCheckpointToStream(DBCheckpoint checkpoint, } } - private static void includeFile(File file, String entryName, + protected static void includeFile(File file, String entryName, ArchiveOutputStream archiveOutputStream) throws IOException { ArchiveEntry archiveEntry = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 0f8413996d1..60bf55558ea 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -20,17 +20,38 @@ import javax.servlet.ServletException; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorOutputStream; +import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.recon.ReconConfig; import org.apache.hadoop.hdds.utils.DBCheckpointServlet; +import org.apache.hadoop.hdds.utils.db.DBCheckpoint; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Provides the current checkpoint Snapshot of the OM DB. (tar.gz) @@ -50,6 +71,7 @@ public class OMDBCheckpointServlet extends DBCheckpointServlet { private static final Logger LOG = LoggerFactory.getLogger(OMDBCheckpointServlet.class); private static final long serialVersionUID = 1L; + private static final int MAX_SECONDS_TO_WAIT = 10; @Override public void init() throws ServletException { @@ -82,4 +104,120 @@ public void init() throws ServletException { allowedGroups, om.isSpnegoEnabled()); } + + private void getFilesFromCheckpoint(DBCheckpoint checkpoint, + Map noCopyFiles) + throws IOException { + Path dir = checkpoint.getCheckpointLocation(); + + try (Stream files = Files.list(dir)) { + for(Path file : files.collect(Collectors.toList())) { + // get the inode + Object key = Files.readAttributes( + file, BasicFileAttributes.class).fileKey(); + noCopyFiles.put(key, file); + } + } + } + private List getSnapshotDirs(DBCheckpoint checkpoint) + throws IOException { + ArrayList list = new ArrayList<>(); + + // get snapshotInfo entries + OzoneConfiguration conf = ((OzoneManager) getServletContext() + .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) + .getConfiguration(); + + OmMetadataManagerImpl checkpointMetadataManager = + OmMetadataManagerImpl.createSnapshotMetadataManager( + conf, checkpoint.getCheckpointLocation().toString()); + try(TableIterator> + iterator = checkpointMetadataManager + .getSnapshotInfoTable().iterator()) { + + // add each entries directory to the list + while (iterator.hasNext()) { + Table.KeyValue entry = iterator.next(); + Path path = Paths.get(entry.getValue().getCheckpointDirName()); + list.add(path); + } + } + return list; + } + + private void waitForDirToExist(Path dir) + throws IOException, InterruptedException { + int count = 0; + while (!dir.toFile().exists() && count < MAX_SECONDS_TO_WAIT) { + Thread.sleep(1000); + count++; + } + if (count == MAX_SECONDS_TO_WAIT) { + throw new IOException("snapshot dir doesn't exist: " + dir); + } + } + private void processDir(Path dir, Map noCopyFiles, + Map copyFiles, + Map hardLinkFiles) + throws IOException, InterruptedException { + waitForDirToExist(dir); + try (Stream files = Files.list(dir)) { + for(Path file : files.collect(Collectors.toList())) { + // get the inode + Object key = Files.readAttributes( + file, BasicFileAttributes.class).fileKey(); + if (noCopyFiles.containsKey(key)) { + hardLinkFiles.put(file, noCopyFiles.get(key)); + } else if (copyFiles.containsKey(key)) { + hardLinkFiles.put(file, copyFiles.get(key)); + } else { + copyFiles.put(key, file); + } + } + } + + } + public void createTarBall(DBCheckpoint checkpoint, OutputStream destination) + throws IOException, InterruptedException, CompressorException { + HashMap noCopyFiles = new HashMap<>(); + HashMap copyFiles = new HashMap<>(); + HashMap hardLinkFiles = new HashMap<>(); + getFilesFromCheckpoint(checkpoint, noCopyFiles); + for (Path dir : getSnapshotDirs(checkpoint)) { + processDir(dir, noCopyFiles, copyFiles, hardLinkFiles); + } + try (CompressorOutputStream gzippedOut = new CompressorStreamFactory() + .createCompressorOutputStream(CompressorStreamFactory.GZIP, + destination)) { + + try (ArchiveOutputStream archiveOutputStream = + new TarArchiveOutputStream(gzippedOut)) { + + Path checkpointPath = checkpoint.getCheckpointLocation(); + try (Stream files = Files.list(checkpointPath)) { + for (Path path : files.collect(Collectors.toList())) { + if (path != null) { + Path fileName = path.getFileName(); + if (fileName != null) { + includeFile(path.toFile(), fileName.toString(), + archiveOutputStream); + } + } + } + } + for (Path file : copyFiles.values()) { + includeFile(file.toFile(), file.getFileName().toString(), + archiveOutputStream); + } + Path hardLinkList = createHardLinkList(checkpointPath, hardLinkFiles); + includeFile(hardLinkList.toFile(), hardLinkList.getFileName().toString(), + archiveOutputStream); + } + } catch (CompressorException e) { + throw new IOException( + "Can't compress the checkpoint: " + + checkpoint.getCheckpointLocation(), e); + } + } + } From 0cc7a2b2cfde8bb5275a5553a4026013d98bfd83 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 5 Oct 2022 15:25:08 -0700 Subject: [PATCH 002/120] added files to tarball --- .../ozone/om/OMDBCheckpointServlet.java | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 60bf55558ea..56b8d994b6c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -106,7 +107,7 @@ public void init() throws ServletException { } private void getFilesFromCheckpoint(DBCheckpoint checkpoint, - Map noCopyFiles) + Map copyFiles) throws IOException { Path dir = checkpoint.getCheckpointLocation(); @@ -115,7 +116,7 @@ private void getFilesFromCheckpoint(DBCheckpoint checkpoint, // get the inode Object key = Files.readAttributes( file, BasicFileAttributes.class).fileKey(); - noCopyFiles.put(key, file); + copyFiles.put(key, file); } } } @@ -156,8 +157,7 @@ private void waitForDirToExist(Path dir) throw new IOException("snapshot dir doesn't exist: " + dir); } } - private void processDir(Path dir, Map noCopyFiles, - Map copyFiles, + private void processDir(Path dir, Map copyFiles, Map hardLinkFiles) throws IOException, InterruptedException { waitForDirToExist(dir); @@ -166,9 +166,7 @@ private void processDir(Path dir, Map noCopyFiles, // get the inode Object key = Files.readAttributes( file, BasicFileAttributes.class).fileKey(); - if (noCopyFiles.containsKey(key)) { - hardLinkFiles.put(file, noCopyFiles.get(key)); - } else if (copyFiles.containsKey(key)) { + if (copyFiles.containsKey(key)) { hardLinkFiles.put(file, copyFiles.get(key)); } else { copyFiles.put(key, file); @@ -177,15 +175,17 @@ private void processDir(Path dir, Map noCopyFiles, } } + public void createTarBall(DBCheckpoint checkpoint, OutputStream destination) throws IOException, InterruptedException, CompressorException { - HashMap noCopyFiles = new HashMap<>(); HashMap copyFiles = new HashMap<>(); HashMap hardLinkFiles = new HashMap<>(); - getFilesFromCheckpoint(checkpoint, noCopyFiles); + + getFilesFromCheckpoint(checkpoint, copyFiles); for (Path dir : getSnapshotDirs(checkpoint)) { - processDir(dir, noCopyFiles, copyFiles, hardLinkFiles); + processDir(dir, copyFiles, hardLinkFiles); } + try (CompressorOutputStream gzippedOut = new CompressorStreamFactory() .createCompressorOutputStream(CompressorStreamFactory.GZIP, destination)) { @@ -194,23 +194,15 @@ public void createTarBall(DBCheckpoint checkpoint, OutputStream destination) new TarArchiveOutputStream(gzippedOut)) { Path checkpointPath = checkpoint.getCheckpointLocation(); - try (Stream files = Files.list(checkpointPath)) { - for (Path path : files.collect(Collectors.toList())) { - if (path != null) { - Path fileName = path.getFileName(); - if (fileName != null) { - includeFile(path.toFile(), fileName.toString(), - archiveOutputStream); - } - } - } - } + for (Path file : copyFiles.values()) { includeFile(file.toFile(), file.getFileName().toString(), archiveOutputStream); } - Path hardLinkList = createHardLinkList(checkpointPath, hardLinkFiles); - includeFile(hardLinkList.toFile(), hardLinkList.getFileName().toString(), + Path hardLinkFile = createHardLinkList(hardLinkFiles); + includeFile(hardLinkFile.toFile(), + Paths.get(checkpointPath.toString(), + "hardLinkFile").toString(), archiveOutputStream); } } catch (CompressorException e) { @@ -220,4 +212,18 @@ public void createTarBall(DBCheckpoint checkpoint, OutputStream destination) } } + private Path createHardLinkList(Map hardLinkFiles) + throws IOException { + Path data = Files.createTempFile("hardLinkData", "txt"); + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : hardLinkFiles.entrySet()) { + sb.append(entry.getKey()) + .append("\t") + .append(entry.getValue()) + .append("\n"); + } + Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); + return data; + } + } From b386cd46faedccb995b4ddbb1946e97816e69901 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 6 Oct 2022 10:03:38 -0700 Subject: [PATCH 003/120] added test --- .../ozone/om/TestOMDbCheckpointServlet.java | 57 +++++++++++++++++++ .../ozone/om/OMDBCheckpointServlet.java | 2 +- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 90743f2e17f..0267c046dc2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -23,10 +23,12 @@ import javax.servlet.WriteListener; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -35,12 +37,20 @@ import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.UUID; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.security.UserGroupInformation; import org.apache.commons.io.FileUtils; @@ -64,6 +74,7 @@ import org.junit.rules.Timeout; import org.mockito.Matchers; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; @@ -83,6 +94,8 @@ public class TestOMDbCheckpointServlet { private HttpServletRequest requestMock = null; private HttpServletResponse responseMock = null; private OMDBCheckpointServlet omDbCheckpointServletMock = null; + private OzoneManagerProtocol writeClient; + private DBCheckpoint dbCheckpoint; @Rule public Timeout timeout = Timeout.seconds(240); @@ -138,6 +151,9 @@ private void setupCluster() throws Exception { .setNumDatanodes(1) .build(); cluster.waitForClusterToBeReady(); + writeClient = cluster.getRpcClient().getObjectStore() + .getClientProxy().getOzoneManagerClient(); + omMetrics = cluster.getOzoneManager().getMetrics(); omDbCheckpointServletMock = @@ -162,6 +178,20 @@ private void setupCluster() throws Exception { doCallRealMethod().when(omDbCheckpointServletMock).doGet(requestMock, responseMock); + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + "content"); + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + "content"); + Thread.sleep(2000); + writeClient.createSnapshot(bucket.getVolumeName(), bucket.getName(), + UUID.randomUUID().toString()); + Thread.sleep(2000); + dbCheckpoint = cluster.getOzoneManager() + .getMetadataManager().getStore() + .getCheckpoint(true); + doCallRealMethod().when(omDbCheckpointServletMock) + .writeArchiveToStream(any(), any()); } @Test @@ -290,6 +320,33 @@ public void testWriteCheckpointToOutputStream() throws Exception { IOUtils.closeStream(fos); } } + @Test + public void testWriteArchiveToStream() + throws Exception { + setupCluster(); + tempFile = File.createTempFile("testWriteArchiveToStream_" + System + .currentTimeMillis(), ".tar.gz"); + + FileOutputStream fileOutputStream = new FileOutputStream(tempFile); + omDbCheckpointServletMock.writeArchiveToStream(dbCheckpoint, + fileOutputStream); + fileOutputStream.close(); + + + TarArchiveInputStream tarInput = new TarArchiveInputStream(new GzipCompressorInputStream(new FileInputStream(tempFile))); + TarArchiveEntry currentEntry = tarInput.getNextTarEntry(); + BufferedReader br = null; + while (currentEntry != null) { + br = new BufferedReader(new InputStreamReader(tarInput)); // Read directly from tarInput + System.out.println("For File = " + currentEntry.getName()); + String line; + while ((line = br.readLine()) != null) { + System.out.println("line="+line); + } + currentEntry = tarInput.getNextTarEntry(); // You forgot to iterate to the next file + } + + } } class TestDBCheckpoint implements DBCheckpoint { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 56b8d994b6c..d074cdcba65 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -176,7 +176,7 @@ private void processDir(Path dir, Map copyFiles, } - public void createTarBall(DBCheckpoint checkpoint, OutputStream destination) + public void writeArchiveToStream(DBCheckpoint checkpoint, OutputStream destination) throws IOException, InterruptedException, CompressorException { HashMap copyFiles = new HashMap<>(); HashMap hardLinkFiles = new HashMap<>(); From 9e19ff9d52544803972d354f927f87aba3e1c6cb Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 6 Oct 2022 14:05:39 -0700 Subject: [PATCH 004/120] basics working --- .../ozone/om/OMDBCheckpointServlet.java | 15 ++++++++---- .../ozone/om/OmMetadataManagerImpl.java | 23 +++++++++++++------ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index d074cdcba65..633a3fcbac9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -54,6 +54,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; + /** * Provides the current checkpoint Snapshot of the OM DB. (tar.gz) * @@ -130,8 +134,8 @@ private List getSnapshotDirs(DBCheckpoint checkpoint) .getConfiguration(); OmMetadataManagerImpl checkpointMetadataManager = - OmMetadataManagerImpl.createSnapshotMetadataManager( - conf, checkpoint.getCheckpointLocation().toString()); + OmMetadataManagerImpl.createCheckpointMetadataManager( + conf, checkpoint); try(TableIterator> iterator = checkpointMetadataManager .getSnapshotInfoTable().iterator()) { @@ -139,7 +143,9 @@ private List getSnapshotDirs(DBCheckpoint checkpoint) // add each entries directory to the list while (iterator.hasNext()) { Table.KeyValue entry = iterator.next(); - Path path = Paths.get(entry.getValue().getCheckpointDirName()); + Path path = Paths.get(OMStorage.getOmDbDir(conf) + + OM_KEY_PREFIX + OM_SNAPSHOT_DIR + OM_KEY_PREFIX + + OM_DB_NAME + entry.getValue().getCheckpointDirName()); list.add(path); } } @@ -192,7 +198,8 @@ public void writeArchiveToStream(DBCheckpoint checkpoint, OutputStream destinati try (ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(gzippedOut)) { - + ((TarArchiveOutputStream)archiveOutputStream) + .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); Path checkpointPath = checkpoint.getCheckpointLocation(); for (Path file : copyFiles.values()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index e78df5a4a54..976da353732 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -18,6 +18,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; @@ -33,6 +34,7 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.hdds.utils.db.RocksDBConfiguration; @@ -298,15 +300,12 @@ protected OmMetadataManagerImpl() { this.omEpoch = 0; } - // metadata constructor for snapshots - private OmMetadataManagerImpl(OzoneConfiguration conf, String snapshotDirName) + // metadata constructor for checkpoints + private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) throws IOException { lock = new OmReadOnlyLock(); omEpoch = 0; - String snapshotDir = OMStorage.getOmDbDir(conf) + - OM_KEY_PREFIX + OM_SNAPSHOT_DIR; - setStore(loadDB(conf, new File(snapshotDir), - OM_DB_NAME + snapshotDirName, true)); + setStore(loadDB(conf, dir, name, true)); initializeOmTables(); } @@ -320,11 +319,21 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, String snapshotDirName) */ public static OmMetadataManagerImpl createSnapshotMetadataManager( OzoneConfiguration conf, String snapshotDirName) throws IOException { + File snapshotDir = new File(OMStorage.getOmDbDir(conf) + + OM_KEY_PREFIX + OM_SNAPSHOT_DIR); OmMetadataManagerImpl smm = new OmMetadataManagerImpl(conf, - snapshotDirName); + snapshotDir, OM_DB_NAME + snapshotDirName); return smm; } + public static OmMetadataManagerImpl createCheckpointMetadataManager( + OzoneConfiguration conf, DBCheckpoint checkpoint) throws IOException { + Path path = checkpoint.getCheckpointLocation(); + File dir = path.getParent().toFile(); + String name = path.getFileName().toString(); + return new OmMetadataManagerImpl(conf, dir, name); + } + @Override public Table getUserTable() { return userTable; From 2dda59c8b687ed3264fb2847810d13f67d4efe7f Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 6 Oct 2022 14:35:00 -0700 Subject: [PATCH 005/120] cleanup --- .../ozone/om/TestOMDbCheckpointServlet.java | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 0267c046dc2..8baf4e2cdf3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -39,10 +39,12 @@ import java.util.LinkedHashSet; import java.util.UUID; +import com.google.common.io.Files; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.CompressorException; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.io.IOUtils; @@ -94,8 +96,6 @@ public class TestOMDbCheckpointServlet { private HttpServletRequest requestMock = null; private HttpServletResponse responseMock = null; private OMDBCheckpointServlet omDbCheckpointServletMock = null; - private OzoneManagerProtocol writeClient; - private DBCheckpoint dbCheckpoint; @Rule public Timeout timeout = Timeout.seconds(240); @@ -151,9 +151,6 @@ private void setupCluster() throws Exception { .setNumDatanodes(1) .build(); cluster.waitForClusterToBeReady(); - writeClient = cluster.getRpcClient().getObjectStore() - .getClientProxy().getOzoneManagerClient(); - omMetrics = cluster.getOzoneManager().getMetrics(); omDbCheckpointServletMock = @@ -178,20 +175,6 @@ private void setupCluster() throws Exception { doCallRealMethod().when(omDbCheckpointServletMock).doGet(requestMock, responseMock); - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), - "content"); - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), - "content"); - Thread.sleep(2000); - writeClient.createSnapshot(bucket.getVolumeName(), bucket.getName(), - UUID.randomUUID().toString()); - Thread.sleep(2000); - dbCheckpoint = cluster.getOzoneManager() - .getMetadataManager().getStore() - .getCheckpoint(true); - doCallRealMethod().when(omDbCheckpointServletMock) - .writeArchiveToStream(any(), any()); } @Test @@ -324,6 +307,24 @@ public void testWriteCheckpointToOutputStream() throws Exception { public void testWriteArchiveToStream() throws Exception { setupCluster(); + OzoneManagerProtocol writeClient = cluster.getRpcClient().getObjectStore() + .getClientProxy().getOzoneManagerClient(); + + + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + "content"); + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + "content"); + Thread.sleep(2000); + writeClient.createSnapshot(bucket.getVolumeName(), bucket.getName(), + UUID.randomUUID().toString()); + Thread.sleep(2000); + DBCheckpoint dbCheckpoint = cluster.getOzoneManager() + .getMetadataManager().getStore() + .getCheckpoint(true); + doCallRealMethod().when(omDbCheckpointServletMock) + .writeArchiveToStream(any(), any()); tempFile = File.createTempFile("testWriteArchiveToStream_" + System .currentTimeMillis(), ".tar.gz"); @@ -331,20 +332,9 @@ public void testWriteArchiveToStream() omDbCheckpointServletMock.writeArchiveToStream(dbCheckpoint, fileOutputStream); fileOutputStream.close(); - - - TarArchiveInputStream tarInput = new TarArchiveInputStream(new GzipCompressorInputStream(new FileInputStream(tempFile))); - TarArchiveEntry currentEntry = tarInput.getNextTarEntry(); - BufferedReader br = null; - while (currentEntry != null) { - br = new BufferedReader(new InputStreamReader(tarInput)); // Read directly from tarInput - System.out.println("For File = " + currentEntry.getName()); - String line; - while ((line = br.readLine()) != null) { - System.out.println("line="+line); - } - currentEntry = tarInput.getNextTarEntry(); // You forgot to iterate to the next file - } + String testDirName = folder.newFolder().getAbsolutePath(); + FileUtil.unTar(tempFile, new File(testDirName)); + Thread.sleep(1000); } } From bfd3054bf2a2eb53e0d19ad8aba337f86d862c1b Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 6 Oct 2022 16:28:39 -0700 Subject: [PATCH 006/120] basically working --- .../hdds/utils/DBCheckpointServlet.java | 8 ++++- .../ozone/om/TestOMDbCheckpointServlet.java | 7 ++-- .../ozone/om/OMDBCheckpointServlet.java | 34 ++++++++++++++----- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index a6a07beb6fc..abbcde74c49 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -174,7 +174,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { file.toString() + ".tgz\""); Instant start = Instant.now(); - writeDBCheckpointToStream(checkpoint, + returnDBCheckpointToStream(checkpoint, response.getOutputStream()); Instant end = Instant.now(); @@ -200,6 +200,12 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { } } + protected void returnDBCheckpointToStream(DBCheckpoint checkpoint, + OutputStream destination) + throws IOException, InterruptedException, CompressorException { + writeDBCheckpointToStream(checkpoint, destination); + } + /** * Write DB Checkpoint to an output stream as a compressed file (tgz). * diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 8baf4e2cdf3..46552917566 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -324,18 +324,17 @@ public void testWriteArchiveToStream() .getMetadataManager().getStore() .getCheckpoint(true); doCallRealMethod().when(omDbCheckpointServletMock) - .writeArchiveToStream(any(), any()); + .returnDBCheckpointToStream(any(), any()); tempFile = File.createTempFile("testWriteArchiveToStream_" + System .currentTimeMillis(), ".tar.gz"); FileOutputStream fileOutputStream = new FileOutputStream(tempFile); - omDbCheckpointServletMock.writeArchiveToStream(dbCheckpoint, + omDbCheckpointServletMock.returnDBCheckpointToStream(dbCheckpoint, fileOutputStream); fileOutputStream.close(); String testDirName = folder.newFolder().getAbsolutePath(); FileUtil.unTar(tempFile, new File(testDirName)); - Thread.sleep(1000); - + System.out.println("gbjtestdir is: " + testDirName); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 633a3fcbac9..bf8d228e015 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -27,6 +27,7 @@ import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.recon.ReconConfig; +import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.DBCheckpointServlet; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.Table; @@ -38,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; @@ -51,6 +53,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -182,11 +185,21 @@ private void processDir(Path dir, Map copyFiles, } - public void writeArchiveToStream(DBCheckpoint checkpoint, OutputStream destination) + @Override + public void returnDBCheckpointToStream(DBCheckpoint checkpoint, OutputStream destination) throws IOException, InterruptedException, CompressorException { HashMap copyFiles = new HashMap<>(); HashMap hardLinkFiles = new HashMap<>(); + OzoneConfiguration conf = ((OzoneManager) getServletContext() + .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) + .getConfiguration(); + + File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); + int truncateLength = metaDirPath.toString().length(); + String fixedDir = fixFile(truncateLength, + checkpoint.getCheckpointLocation()); + getFilesFromCheckpoint(checkpoint, copyFiles); for (Path dir : getSnapshotDirs(checkpoint)) { processDir(dir, copyFiles, hardLinkFiles); @@ -203,13 +216,13 @@ public void writeArchiveToStream(DBCheckpoint checkpoint, OutputStream destinati Path checkpointPath = checkpoint.getCheckpointLocation(); for (Path file : copyFiles.values()) { - includeFile(file.toFile(), file.getFileName().toString(), + String fixedFile = fixFile(truncateLength, file); + includeFile(file.toFile(), fixedFile, archiveOutputStream); } - Path hardLinkFile = createHardLinkList(hardLinkFiles); + Path hardLinkFile = createHardLinkList(truncateLength, hardLinkFiles); includeFile(hardLinkFile.toFile(), - Paths.get(checkpointPath.toString(), - "hardLinkFile").toString(), + Paths.get(fixedDir, "hardLinkFile").toString(), archiveOutputStream); } } catch (CompressorException e) { @@ -219,18 +232,23 @@ public void writeArchiveToStream(DBCheckpoint checkpoint, OutputStream destinati } } - private Path createHardLinkList(Map hardLinkFiles) + + private Path createHardLinkList(int truncateLength, + Map hardLinkFiles) throws IOException { Path data = Files.createTempFile("hardLinkData", "txt"); StringBuilder sb = new StringBuilder(); for (Map.Entry entry : hardLinkFiles.entrySet()) { - sb.append(entry.getKey()) + sb.append(fixFile(truncateLength, entry.getKey())) .append("\t") - .append(entry.getValue()) + .append(fixFile(truncateLength, entry.getValue())) .append("\n"); } Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); return data; } + private String fixFile(int truncateLength, Path file) { + return file.toString().substring(truncateLength); + } } From 245c740faf6673b1feb88cb355b66fdfa2389dde Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 7 Oct 2022 11:11:15 -0700 Subject: [PATCH 007/120] fix original tests --- .../ozone/om/TestOMDbCheckpointServlet.java | 70 +------------------ 1 file changed, 3 insertions(+), 67 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 46552917566..e664ebdc70a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -175,6 +175,9 @@ private void setupCluster() throws Exception { doCallRealMethod().when(omDbCheckpointServletMock).doGet(requestMock, responseMock); + + doCallRealMethod().when(omDbCheckpointServletMock) + .returnDBCheckpointToStream(any(), any()); } @Test @@ -271,38 +274,6 @@ public void testSpnegoEnabled() throws Exception { Assert.assertTrue(tempFile.length() > 0); } - @Test - public void testWriteCheckpointToOutputStream() throws Exception { - - FileInputStream fis = null; - FileOutputStream fos = null; - - try { - String testDirName = folder.newFolder().getAbsolutePath(); - File file = new File(testDirName + "/temp1.txt"); - OutputStreamWriter writer = new OutputStreamWriter( - new FileOutputStream(file), StandardCharsets.UTF_8); - writer.write("Test data 1"); - writer.close(); - - file = new File(testDirName + "/temp2.txt"); - writer = new OutputStreamWriter( - new FileOutputStream(file), StandardCharsets.UTF_8); - writer.write("Test data 2"); - writer.close(); - - File outputFile = - new File(Paths.get(testDirName, "output_file.tgz").toString()); - TestDBCheckpoint dbCheckpoint = new TestDBCheckpoint( - Paths.get(testDirName)); - writeDBCheckpointToStream(dbCheckpoint, - new FileOutputStream(outputFile)); - assertNotNull(outputFile); - } finally { - IOUtils.closeStream(fis); - IOUtils.closeStream(fos); - } - } @Test public void testWriteArchiveToStream() throws Exception { @@ -323,8 +294,6 @@ public void testWriteArchiveToStream() DBCheckpoint dbCheckpoint = cluster.getOzoneManager() .getMetadataManager().getStore() .getCheckpoint(true); - doCallRealMethod().when(omDbCheckpointServletMock) - .returnDBCheckpointToStream(any(), any()); tempFile = File.createTempFile("testWriteArchiveToStream_" + System .currentTimeMillis(), ".tar.gz"); @@ -338,36 +307,3 @@ public void testWriteArchiveToStream() } } -class TestDBCheckpoint implements DBCheckpoint { - - private final Path checkpointFile; - - TestDBCheckpoint(Path checkpointFile) { - this.checkpointFile = checkpointFile; - } - - @Override - public Path getCheckpointLocation() { - return checkpointFile; - } - - @Override - public long getCheckpointTimestamp() { - return 0; - } - - @Override - public long getLatestSequenceNumber() { - return 0; - } - - @Override - public long checkpointCreationTimeTaken() { - return 0; - } - - @Override - public void cleanupCheckpoint() throws IOException { - FileUtils.deleteDirectory(checkpointFile.toFile()); - } -} \ No newline at end of file From c7c5b622336457ec0171841903c362ac63c1dfff Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 7 Oct 2022 11:32:54 -0700 Subject: [PATCH 008/120] added create snapshot --- .../ozone/om/TestOMDbCheckpointServlet.java | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index e664ebdc70a..cc68cdc4fcb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -23,35 +23,24 @@ import javax.servlet.WriteListener; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.nio.file.Paths; import java.security.Principal; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.UUID; +import java.util.concurrent.TimeoutException; -import com.google.common.io.Files; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.compressors.CompressorException; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.security.UserGroupInformation; @@ -61,14 +50,17 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; +import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE; -import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.writeDBCheckpointToStream; + +import org.apache.ozone.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; -import static org.junit.Assert.assertNotNull; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -278,32 +270,49 @@ public void testSpnegoEnabled() throws Exception { public void testWriteArchiveToStream() throws Exception { setupCluster(); - OzoneManagerProtocol writeClient = cluster.getRpcClient().getObjectStore() - .getClientProxy().getOzoneManagerClient(); - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), "content"); TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), "content"); + + // this sleep can be removed after this is fixed: + // https://issues.apache.org/jira/browse/HDDS-7279 Thread.sleep(2000); - writeClient.createSnapshot(bucket.getVolumeName(), bucket.getName(), - UUID.randomUUID().toString()); - Thread.sleep(2000); + String snapshotDirName = + createSnapshot(bucket.getVolumeName(), bucket.getName()); DBCheckpoint dbCheckpoint = cluster.getOzoneManager() .getMetadataManager().getStore() .getCheckpoint(true); - tempFile = File.createTempFile("testWriteArchiveToStream_" + System - .currentTimeMillis(), ".tar.gz"); FileOutputStream fileOutputStream = new FileOutputStream(tempFile); omDbCheckpointServletMock.returnDBCheckpointToStream(dbCheckpoint, fileOutputStream); - fileOutputStream.close(); + String testDirName = folder.newFolder().getAbsolutePath(); FileUtil.unTar(tempFile, new File(testDirName)); System.out.println("gbjtestdir is: " + testDirName); } -} + private String createSnapshot(String vname, String bname) + throws IOException, InterruptedException, TimeoutException { + final OzoneManager om = cluster.getOzoneManager(); + File metaDir = OMStorage.getOmDbDir(conf); + String snapshotName = UUID.randomUUID().toString(); + OzoneManagerProtocol writeClient = cluster.getRpcClient().getObjectStore() + .getClientProxy().getOzoneManagerClient(); + + writeClient.createSnapshot(vname, bname, snapshotName); + SnapshotInfo snapshotInfo = om + .getMetadataManager().getSnapshotInfoTable() + .get(SnapshotInfo.getTableKey(vname, bname, snapshotName)); + String snapshotDirName = metaDir + OM_KEY_PREFIX + + OM_SNAPSHOT_DIR + OM_KEY_PREFIX + OM_DB_NAME + + snapshotInfo.getCheckpointDirName() + OM_KEY_PREFIX; + GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), + 100, 2000); + return snapshotDirName; + } + +} From 0c242f4fa5382916da7cf0bf541ed38907404522 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 7 Oct 2022 13:43:36 -0700 Subject: [PATCH 009/120] added checkpoint test --- .../ozone/om/TestOMDbCheckpointServlet.java | 53 +++++++++++++++++-- .../ozone/om/OMDBCheckpointServlet.java | 3 +- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index cc68cdc4fcb..6161da92805 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -26,12 +26,20 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.Principal; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.HashSet; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -68,6 +76,7 @@ import org.junit.rules.Timeout; import org.mockito.Matchers; +import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.fixFile; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doNothing; @@ -88,6 +97,7 @@ public class TestOMDbCheckpointServlet { private HttpServletRequest requestMock = null; private HttpServletResponse responseMock = null; private OMDBCheckpointServlet omDbCheckpointServletMock = null; + private File metaDir; @Rule public Timeout timeout = Timeout.seconds(240); @@ -105,6 +115,7 @@ public class TestOMDbCheckpointServlet { public void init() throws Exception { conf = new OzoneConfiguration(); + tempFile = File.createTempFile("testDoGet_" + System .currentTimeMillis(), ".tar.gz"); @@ -144,6 +155,7 @@ private void setupCluster() throws Exception { .build(); cluster.waitForClusterToBeReady(); omMetrics = cluster.getOzoneManager().getMetrics(); + metaDir = OMStorage.getOmDbDir(conf); omDbCheckpointServletMock = mock(OMDBCheckpointServlet.class); @@ -282,6 +294,14 @@ public void testWriteArchiveToStream() Thread.sleep(2000); String snapshotDirName = createSnapshot(bucket.getVolumeName(), bucket.getName()); + String snapshotDirName2 = + createSnapshot(bucket.getVolumeName(), bucket.getName()); + + Path dummyFile = Paths.get(snapshotDirName, "dummyFile"); + Path dummyLink = Paths.get(snapshotDirName2, "dummyFile"); + Files.write(dummyFile, "dummyData".getBytes(StandardCharsets.UTF_8)); + Files.createLink(dummyLink, dummyFile); + DBCheckpoint dbCheckpoint = cluster.getOzoneManager() .getMetadataManager().getStore() .getCheckpoint(true); @@ -291,14 +311,41 @@ public void testWriteArchiveToStream() fileOutputStream); String testDirName = folder.newFolder().getAbsolutePath(); + int testDirLength = testDirName.length(); FileUtil.unTar(tempFile, new File(testDirName)); - System.out.println("gbjtestdir is: " + testDirName); - } + Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); + int metaDirLength = metaDir.toString().length(); + String shortCheckpointLocation = + fixFile(metaDirLength, checkpointLocation); + Path finalCheckpointLocation = + Paths.get(testDirName, shortCheckpointLocation); + + Set initialCheckpointSet = new HashSet<>(); + try (Stream files = Files.list(checkpointLocation)) { + for (Path file : files.collect(Collectors.toList())) { + initialCheckpointSet.add(fixFile(metaDirLength, file)); + } + } + Set finalCheckpointSet = new HashSet<>(); + try (Stream files = Files.list(finalCheckpointLocation)) { + for (Path file : files.collect(Collectors.toList())) { + finalCheckpointSet.add(fixFile(testDirLength, file)); + } + } + + String hlString = Paths.get(shortCheckpointLocation, + "hardLinkFile").toString(); + Assert.assertTrue(finalCheckpointSet.contains(hlString)); + + finalCheckpointSet.remove(hlString); + + Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); + + } private String createSnapshot(String vname, String bname) throws IOException, InterruptedException, TimeoutException { final OzoneManager om = cluster.getOzoneManager(); - File metaDir = OMStorage.getOmDbDir(conf); String snapshotName = UUID.randomUUID().toString(); OzoneManagerProtocol writeClient = cluster.getRpcClient().getObjectStore() .getClientProxy().getOzoneManagerClient(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index bf8d228e015..6ccdfc75eed 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -247,7 +247,8 @@ private Path createHardLinkList(int truncateLength, Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); return data; } - private String fixFile(int truncateLength, Path file) { + + static String fixFile(int truncateLength, Path file) { return file.toString().substring(truncateLength); } From fd5ce9bfc9e92697fc8e2a990ac08f8eb901d127 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 7 Oct 2022 15:04:00 -0700 Subject: [PATCH 010/120] added link file test --- .../ozone/om/TestOMDbCheckpointServlet.java | 90 ++++++++++++++++++- 1 file changed, 87 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 6161da92805..8e1170faede 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -333,15 +333,99 @@ public void testWriteArchiveToStream() } } - String hlString = Paths.get(shortCheckpointLocation, + String hlPath = Paths.get(shortCheckpointLocation, "hardLinkFile").toString(); - Assert.assertTrue(finalCheckpointSet.contains(hlString)); + Assert.assertTrue(finalCheckpointSet.contains(hlPath)); - finalCheckpointSet.remove(hlString); + finalCheckpointSet.remove(hlPath); Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); + String shortSnapshotLocation = + fixFile(metaDirLength, Paths.get(snapshotDirName)); + String shortSnapshotLocation2 = + fixFile(metaDirLength, Paths.get(snapshotDirName2)); + + Path finalSnapshotLocation = + Paths.get(testDirName, shortSnapshotLocation); + + Assert.assertTrue(Paths + .get(finalSnapshotLocation.toString(), + "CURRENT").toFile().exists()); + + Set initialSnapshotSet = new HashSet<>(); + try (Stream files = Files.list(Paths.get(snapshotDirName))) { + for (Path file : files.collect(Collectors.toList())) { + if (!file.getFileName().toString().equals("dummyFile")) { + initialSnapshotSet.add(fixFile(metaDirLength, file)); + } + } + } + Set finalSnapshotSet = new HashSet<>(); + boolean foundManifest = false; + try (Stream files = Files.list(finalSnapshotLocation)) { + for (Path file : files.collect(Collectors.toList())) { + if (file.toString().contains("MANIFEST")) { + foundManifest = true; + } + finalSnapshotSet.add(fixFile(testDirLength, file)); + } + } + Assert.assertTrue("snapshot manifest found", foundManifest); + + Stream lines = Files.lines(Paths.get(testDirName, hlPath)); + boolean linesFound = false; + boolean dummyLinkFound = false; + for (String line: lines.collect(Collectors.toList())) { + linesFound = true; + if (line.contains("dummyFile")) { + dummyLinkFound = true; + Assert.assertTrue(checkDummyFile(shortSnapshotLocation,shortSnapshotLocation2, line)); + } else { + Assert.assertTrue(checkLine(shortSnapshotLocation, shortSnapshotLocation2, shortCheckpointLocation, line)); + if (line.startsWith(shortSnapshotLocation)) { + finalSnapshotSet.add(line.split("\t")[0]); + } + } + } + Assert.assertTrue("hard link file not empty", linesFound); + Assert.assertTrue("dummy link found", dummyLinkFound); + Assert.assertEquals(initialSnapshotSet, finalSnapshotSet); + + } + private boolean checkDummyFile(String dir0, String dir1, String line) { + String[] files = line.split("\t"); + if (!files[0].startsWith(dir0) && !files[1].startsWith(dir0)) { + return false; + } + if (!files[0].startsWith(dir1) && !files[1].startsWith(dir1)) { + return false; + } + Path path0 = Paths.get(files[0]); + Path path1 = Paths.get(files[1]); + if (path0.getParent().equals(path1.getParent())) { + return false; + } + return path0.getFileName().equals(path1.getFileName()); + } + private boolean checkLine(String shortSnapshotLocation, + String shortSnapshotLocation2, + String shortCheckpointLocation, String line) { + String[] files = line.split("\t"); + if (!files[0].startsWith(shortSnapshotLocation) && + !files[0].startsWith(shortSnapshotLocation2)) { + return false; + } + if (!files[1].startsWith(shortCheckpointLocation)) { + return false; + } + String file0 = files[0].substring(shortSnapshotLocation.length()); + String file1 = files[1].substring(shortCheckpointLocation.length()); + if (Paths.get(file0).getNameCount() > 1) { + return false; + } + return file0.equals(file1); } private String createSnapshot(String vname, String bname) throws IOException, InterruptedException, TimeoutException { From b1435657500f7ced8036f841ca583a57273fff15 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 7 Oct 2022 15:18:48 -0700 Subject: [PATCH 011/120] cleanup --- .../ozone/om/TestOMDbCheckpointServlet.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 8e1170faede..396750dcc77 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -115,7 +115,6 @@ public class TestOMDbCheckpointServlet { public void init() throws Exception { conf = new OzoneConfiguration(); - tempFile = File.createTempFile("testDoGet_" + System .currentTimeMillis(), ".tar.gz"); @@ -155,7 +154,6 @@ private void setupCluster() throws Exception { .build(); cluster.waitForClusterToBeReady(); omMetrics = cluster.getOzoneManager().getMetrics(); - metaDir = OMStorage.getOmDbDir(conf); omDbCheckpointServletMock = mock(OMDBCheckpointServlet.class); @@ -282,6 +280,7 @@ public void testSpnegoEnabled() throws Exception { public void testWriteArchiveToStream() throws Exception { setupCluster(); + metaDir = OMStorage.getOmDbDir(conf); OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), @@ -297,6 +296,10 @@ public void testWriteArchiveToStream() String snapshotDirName2 = createSnapshot(bucket.getVolumeName(), bucket.getName()); + + // create dummy link from one snapshot dir to the other + // to confirm that links are recognized even if + // they are not in the checkpoint directory Path dummyFile = Paths.get(snapshotDirName, "dummyFile"); Path dummyLink = Paths.get(snapshotDirName2, "dummyFile"); Files.write(dummyFile, "dummyData".getBytes(StandardCharsets.UTF_8)); @@ -310,9 +313,12 @@ public void testWriteArchiveToStream() omDbCheckpointServletMock.returnDBCheckpointToStream(dbCheckpoint, fileOutputStream); + // Untar the file into a temp folder to be examined String testDirName = folder.newFolder().getAbsolutePath(); int testDirLength = testDirName.length(); FileUtil.unTar(tempFile, new File(testDirName)); + + Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); int metaDirLength = metaDir.toString().length(); String shortCheckpointLocation = @@ -320,6 +326,7 @@ public void testWriteArchiveToStream() Path finalCheckpointLocation = Paths.get(testDirName, shortCheckpointLocation); + // Confirm the checkpoint directories match Set initialCheckpointSet = new HashSet<>(); try (Stream files = Files.list(checkpointLocation)) { for (Path file : files.collect(Collectors.toList())) { @@ -333,6 +340,7 @@ public void testWriteArchiveToStream() } } + // Confirm hardLinkFile exists in checkpoint dir String hlPath = Paths.get(shortCheckpointLocation, "hardLinkFile").toString(); Assert.assertTrue(finalCheckpointSet.contains(hlPath)); @@ -350,10 +358,12 @@ public void testWriteArchiveToStream() Path finalSnapshotLocation = Paths.get(testDirName, shortSnapshotLocation); + // Confirm snapshot current file exists and is not a hard link Assert.assertTrue(Paths .get(finalSnapshotLocation.toString(), "CURRENT").toFile().exists()); + // get initial snapshot files Set initialSnapshotSet = new HashSet<>(); try (Stream files = Files.list(Paths.get(snapshotDirName))) { for (Path file : files.collect(Collectors.toList())) { @@ -362,6 +372,8 @@ public void testWriteArchiveToStream() } } } + + // get final snapshot files Set finalSnapshotSet = new HashSet<>(); boolean foundManifest = false; try (Stream files = Files.list(finalSnapshotLocation)) { @@ -374,6 +386,7 @@ public void testWriteArchiveToStream() } Assert.assertTrue("snapshot manifest found", foundManifest); + // check each line in the hard link file Stream lines = Files.lines(Paths.get(testDirName, hlPath)); boolean linesFound = false; boolean dummyLinkFound = false; @@ -391,7 +404,8 @@ public void testWriteArchiveToStream() } Assert.assertTrue("hard link file not empty", linesFound); Assert.assertTrue("dummy link found", dummyLinkFound); - Assert.assertEquals(initialSnapshotSet, finalSnapshotSet); + Assert.assertEquals("found expected snapshot files", + initialSnapshotSet, finalSnapshotSet); } private boolean checkDummyFile(String dir0, String dir1, String line) { From a402d14e0276e2300dd92e4889f1e4ce4c5a7b8f Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 7 Oct 2022 16:04:15 -0700 Subject: [PATCH 012/120] fixed scm checkpoint tests --- .../apache/hadoop/hdds/utils/DBCheckpointServlet.java | 4 ++-- .../hadoop/hdds/scm/TestSCMDbCheckpointServlet.java | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index abbcde74c49..a35174d2a0b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -200,8 +200,8 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { } } - protected void returnDBCheckpointToStream(DBCheckpoint checkpoint, - OutputStream destination) + public void returnDBCheckpointToStream(DBCheckpoint checkpoint, + OutputStream destination) throws IOException, InterruptedException, CompressorException { writeDBCheckpointToStream(checkpoint, destination); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java index 8a91388def4..ce1dea48400 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.UUID; +import org.apache.commons.compress.compressors.CompressorException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics; import org.apache.hadoop.hdds.scm.server.SCMDBCheckpointServlet; @@ -46,6 +47,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.mockito.Matchers; + +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; @@ -99,7 +102,9 @@ public void shutdown() { } @Test - public void testDoGet() throws ServletException, IOException { + public void testDoGet() + throws ServletException, IOException, CompressorException, + InterruptedException { File tempFile = null; try { @@ -114,6 +119,8 @@ public void testDoGet() throws ServletException, IOException { Collections.emptyList(), Collections.emptyList(), false); + doCallRealMethod().when(scmDbCheckpointServletMock) + .returnDBCheckpointToStream(any(), any()); HttpServletRequest requestMock = mock(HttpServletRequest.class); HttpServletResponse responseMock = mock(HttpServletResponse.class); From af55d3f4c2d8849c7b867ce426b1670477e94c6c Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 7 Oct 2022 18:15:29 -0700 Subject: [PATCH 013/120] reformatted for backwards compatibilty --- .../ozone/om/TestOMDbCheckpointServlet.java | 24 +++++++++---------- .../ozone/om/OMDBCheckpointServlet.java | 22 ++++++++++------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 396750dcc77..a81afcc7ad2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -321,28 +321,28 @@ public void testWriteArchiveToStream() Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); int metaDirLength = metaDir.toString().length(); - String shortCheckpointLocation = - fixFile(metaDirLength, checkpointLocation); Path finalCheckpointLocation = - Paths.get(testDirName, shortCheckpointLocation); + Paths.get(testDirName); // Confirm the checkpoint directories match Set initialCheckpointSet = new HashSet<>(); try (Stream files = Files.list(checkpointLocation)) { for (Path file : files.collect(Collectors.toList())) { - initialCheckpointSet.add(fixFile(metaDirLength, file)); + initialCheckpointSet.add(fixFile(checkpointLocation.toString().length(), file)); } } Set finalCheckpointSet = new HashSet<>(); try (Stream files = Files.list(finalCheckpointLocation)) { for (Path file : files.collect(Collectors.toList())) { + if (file.getFileName().toString().equals("db.snapshots")) { + continue; + } finalCheckpointSet.add(fixFile(testDirLength, file)); } } // Confirm hardLinkFile exists in checkpoint dir - String hlPath = Paths.get(shortCheckpointLocation, - "hardLinkFile").toString(); + String hlPath = "/hardLinkFile"; Assert.assertTrue(finalCheckpointSet.contains(hlPath)); finalCheckpointSet.remove(hlPath); @@ -396,7 +396,7 @@ public void testWriteArchiveToStream() dummyLinkFound = true; Assert.assertTrue(checkDummyFile(shortSnapshotLocation,shortSnapshotLocation2, line)); } else { - Assert.assertTrue(checkLine(shortSnapshotLocation, shortSnapshotLocation2, shortCheckpointLocation, line)); + Assert.assertTrue(checkLine(shortSnapshotLocation, shortSnapshotLocation2, line)); if (line.startsWith(shortSnapshotLocation)) { finalSnapshotSet.add(line.split("\t")[0]); } @@ -425,21 +425,19 @@ private boolean checkDummyFile(String dir0, String dir1, String line) { } private boolean checkLine(String shortSnapshotLocation, String shortSnapshotLocation2, - String shortCheckpointLocation, String line) { + String line) { String[] files = line.split("\t"); if (!files[0].startsWith(shortSnapshotLocation) && !files[0].startsWith(shortSnapshotLocation2)) { return false; } - if (!files[1].startsWith(shortCheckpointLocation)) { - return false; - } + String file0 = files[0].substring(shortSnapshotLocation.length()); - String file1 = files[1].substring(shortCheckpointLocation.length()); + String file1 = files[1]; if (Paths.get(file0).getNameCount() > 1) { return false; } - return file0.equals(file1); + return file0.equals("/" + file1); } private String createSnapshot(String vname, String bname) throws IOException, InterruptedException, TimeoutException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 6ccdfc75eed..db60ba765a4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -53,7 +53,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -197,8 +196,6 @@ public void returnDBCheckpointToStream(DBCheckpoint checkpoint, OutputStream des File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); int truncateLength = metaDirPath.toString().length(); - String fixedDir = fixFile(truncateLength, - checkpoint.getCheckpointLocation()); getFilesFromCheckpoint(checkpoint, copyFiles); for (Path dir : getSnapshotDirs(checkpoint)) { @@ -213,17 +210,20 @@ public void returnDBCheckpointToStream(DBCheckpoint checkpoint, OutputStream des new TarArchiveOutputStream(gzippedOut)) { ((TarArchiveOutputStream)archiveOutputStream) .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - Path checkpointPath = checkpoint.getCheckpointLocation(); for (Path file : copyFiles.values()) { String fixedFile = fixFile(truncateLength, file); + if (fixedFile.startsWith("/db.checkpoints")) { + fixedFile = Paths.get(fixedFile).getFileName().toString(); + } includeFile(file.toFile(), fixedFile, archiveOutputStream); } - Path hardLinkFile = createHardLinkList(truncateLength, hardLinkFiles); - includeFile(hardLinkFile.toFile(), - Paths.get(fixedDir, "hardLinkFile").toString(), - archiveOutputStream); + if (!hardLinkFiles.isEmpty()) { + Path hardLinkFile = createHardLinkList(truncateLength, hardLinkFiles); + includeFile(hardLinkFile.toFile(), "hardLinkFile", + archiveOutputStream); + } } } catch (CompressorException e) { throw new IOException( @@ -239,9 +239,13 @@ private Path createHardLinkList(int truncateLength, Path data = Files.createTempFile("hardLinkData", "txt"); StringBuilder sb = new StringBuilder(); for (Map.Entry entry : hardLinkFiles.entrySet()) { + String fixedFile = fixFile(truncateLength, entry.getValue()); + if (fixedFile.startsWith("/db.checkpoints")) { + fixedFile = Paths.get(fixedFile).getFileName().toString(); + } sb.append(fixFile(truncateLength, entry.getKey())) .append("\t") - .append(fixFile(truncateLength, entry.getValue())) + .append(fixedFile) .append("\n"); } Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); From 77e0b02a2196d9adddcd686840598afcb8477255 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 10 Oct 2022 15:20:49 -0700 Subject: [PATCH 014/120] cleanup --- .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../hdds/utils/DBCheckpointServlet.java | 6 +- .../apache/hadoop/hdds/utils/db/RDBStore.java | 3 +- .../hdds/scm/TestSCMDbCheckpointServlet.java | 2 +- .../ozone/om/TestOMDbCheckpointServlet.java | 21 ++--- .../ozone/om/OMDBCheckpointServlet.java | 79 +++++++++++-------- .../ozone/om/OmMetadataManagerImpl.java | 9 ++- 7 files changed, 73 insertions(+), 48 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 97dd7fde0b5..67544babc12 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -551,6 +551,7 @@ private OzoneConsts() { public static final int OZONE_MAXIMUM_ACCESS_ID_LENGTH = 100; public static final String OM_SNAPSHOT_NAME = "snapshotName"; + public static final String OM_CHECKPOINT_DIR = "db.checkpoints"; public static final String OM_SNAPSHOT_DIR = "db.snapshots"; public static final String OM_SNAPSHOT_INDICATOR = ".snapshot"; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index a35174d2a0b..d46fcfc2bb5 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -174,7 +174,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { file.toString() + ".tgz\""); Instant start = Instant.now(); - returnDBCheckpointToStream(checkpoint, + writeDbDataToStream(checkpoint, response.getOutputStream()); Instant end = Instant.now(); @@ -200,8 +200,8 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { } } - public void returnDBCheckpointToStream(DBCheckpoint checkpoint, - OutputStream destination) + public void writeDbDataToStream(DBCheckpoint checkpoint, + OutputStream destination) throws IOException, InterruptedException, CompressorException { writeDBCheckpointToStream(checkpoint, destination); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 992627c4cfc..e2049b45ec6 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -46,6 +46,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; /** @@ -126,7 +127,7 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, //create checkpoints directory if not exists. checkpointsParentDir = - Paths.get(dbLocation.getParent(), "db.checkpoints").toString(); + Paths.get(dbLocation.getParent(), OM_CHECKPOINT_DIR).toString(); File checkpointsDir = new File(checkpointsParentDir); if (!checkpointsDir.exists()) { boolean success = checkpointsDir.mkdir(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java index ce1dea48400..3398e0d22aa 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java @@ -120,7 +120,7 @@ public void testDoGet() Collections.emptyList(), false); doCallRealMethod().when(scmDbCheckpointServletMock) - .returnDBCheckpointToStream(any(), any()); + .writeDbDataToStream(any(), any()); HttpServletRequest requestMock = mock(HttpServletRequest.class); HttpServletResponse responseMock = mock(HttpServletResponse.class); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index a81afcc7ad2..fe06b54cb29 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -76,7 +76,7 @@ import org.junit.rules.Timeout; import org.mockito.Matchers; -import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.fixFile; +import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.fixFileName; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doNothing; @@ -179,7 +179,7 @@ private void setupCluster() throws Exception { responseMock); doCallRealMethod().when(omDbCheckpointServletMock) - .returnDBCheckpointToStream(any(), any()); + .writeDbDataToStream(any(), any()); } @Test @@ -310,7 +310,7 @@ public void testWriteArchiveToStream() .getCheckpoint(true); FileOutputStream fileOutputStream = new FileOutputStream(tempFile); - omDbCheckpointServletMock.returnDBCheckpointToStream(dbCheckpoint, + omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, fileOutputStream); // Untar the file into a temp folder to be examined @@ -328,7 +328,8 @@ public void testWriteArchiveToStream() Set initialCheckpointSet = new HashSet<>(); try (Stream files = Files.list(checkpointLocation)) { for (Path file : files.collect(Collectors.toList())) { - initialCheckpointSet.add(fixFile(checkpointLocation.toString().length(), file)); + initialCheckpointSet.add( + fixFileName(checkpointLocation.toString().length(), file)); } } Set finalCheckpointSet = new HashSet<>(); @@ -337,7 +338,7 @@ public void testWriteArchiveToStream() if (file.getFileName().toString().equals("db.snapshots")) { continue; } - finalCheckpointSet.add(fixFile(testDirLength, file)); + finalCheckpointSet.add(fixFileName(testDirLength, file)); } } @@ -351,9 +352,9 @@ public void testWriteArchiveToStream() Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); String shortSnapshotLocation = - fixFile(metaDirLength, Paths.get(snapshotDirName)); + fixFileName(metaDirLength, Paths.get(snapshotDirName)); String shortSnapshotLocation2 = - fixFile(metaDirLength, Paths.get(snapshotDirName2)); + fixFileName(metaDirLength, Paths.get(snapshotDirName2)); Path finalSnapshotLocation = Paths.get(testDirName, shortSnapshotLocation); @@ -368,7 +369,7 @@ public void testWriteArchiveToStream() try (Stream files = Files.list(Paths.get(snapshotDirName))) { for (Path file : files.collect(Collectors.toList())) { if (!file.getFileName().toString().equals("dummyFile")) { - initialSnapshotSet.add(fixFile(metaDirLength, file)); + initialSnapshotSet.add(fixFileName(metaDirLength, file)); } } } @@ -381,7 +382,9 @@ public void testWriteArchiveToStream() if (file.toString().contains("MANIFEST")) { foundManifest = true; } - finalSnapshotSet.add(fixFile(testDirLength, file)); + if (!file.getFileName().toString().equals("dummyFile")) { + finalSnapshotSet.add(fixFileName(testDirLength, file)); + } } } Assert.assertTrue("snapshot manifest found", foundManifest); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index db60ba765a4..d381d390bf0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -112,9 +112,10 @@ public void init() throws ServletException { om.isSpnegoEnabled()); } - private void getFilesFromCheckpoint(DBCheckpoint checkpoint, - Map copyFiles) - throws IOException { + private void getFilesForArchive(DBCheckpoint checkpoint, + Map copyFiles, + Map hardLinkFiles) + throws IOException, InterruptedException { Path dir = checkpoint.getCheckpointLocation(); try (Stream files = Files.list(dir)) { @@ -125,7 +126,11 @@ private void getFilesFromCheckpoint(DBCheckpoint checkpoint, copyFiles.put(key, file); } } + for (Path snapshotDir : getSnapshotDirs(checkpoint)) { + processDir(snapshotDir, copyFiles, hardLinkFiles); + } } + private List getSnapshotDirs(DBCheckpoint checkpoint) throws IOException { ArrayList list = new ArrayList<>(); @@ -174,6 +179,7 @@ private void processDir(Path dir, Map copyFiles, // get the inode Object key = Files.readAttributes( file, BasicFileAttributes.class).fileKey(); + // If we already have the inode, store as hard link if (copyFiles.containsKey(key)) { hardLinkFiles.put(file, copyFiles.get(key)); } else { @@ -185,45 +191,25 @@ private void processDir(Path dir, Map copyFiles, } @Override - public void returnDBCheckpointToStream(DBCheckpoint checkpoint, OutputStream destination) + public void writeDbDataToStream(DBCheckpoint checkpoint, OutputStream destination) throws IOException, InterruptedException, CompressorException { + // Map of inodes to path HashMap copyFiles = new HashMap<>(); + // Map of link to path HashMap hardLinkFiles = new HashMap<>(); - OzoneConfiguration conf = ((OzoneManager) getServletContext() - .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) - .getConfiguration(); - - File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); - int truncateLength = metaDirPath.toString().length(); - - getFilesFromCheckpoint(checkpoint, copyFiles); - for (Path dir : getSnapshotDirs(checkpoint)) { - processDir(dir, copyFiles, hardLinkFiles); - } + getFilesForArchive(checkpoint, copyFiles, hardLinkFiles); try (CompressorOutputStream gzippedOut = new CompressorStreamFactory() .createCompressorOutputStream(CompressorStreamFactory.GZIP, destination)) { - try (ArchiveOutputStream archiveOutputStream = + try (TarArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(gzippedOut)) { - ((TarArchiveOutputStream)archiveOutputStream) + archiveOutputStream .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - for (Path file : copyFiles.values()) { - String fixedFile = fixFile(truncateLength, file); - if (fixedFile.startsWith("/db.checkpoints")) { - fixedFile = Paths.get(fixedFile).getFileName().toString(); - } - includeFile(file.toFile(), fixedFile, - archiveOutputStream); - } - if (!hardLinkFiles.isEmpty()) { - Path hardLinkFile = createHardLinkList(truncateLength, hardLinkFiles); - includeFile(hardLinkFile.toFile(), "hardLinkFile", - archiveOutputStream); - } + writeFilesToArchive(copyFiles, hardLinkFiles, archiveOutputStream); } } catch (CompressorException e) { throw new IOException( @@ -232,6 +218,33 @@ public void returnDBCheckpointToStream(DBCheckpoint checkpoint, OutputStream des } } + private void writeFilesToArchive(HashMap copyFiles, + HashMap hardLinkFiles, + ArchiveOutputStream archiveOutputStream) + throws IOException { + + OzoneConfiguration conf = ((OzoneManager) getServletContext() + .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) + .getConfiguration(); + + File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); + int truncateLength = metaDirPath.toString().length(); + + for (Path file : copyFiles.values()) { + String fixedFile = fixFileName(truncateLength, file); + if (fixedFile.startsWith("/db.checkpoints")) { + fixedFile = Paths.get(fixedFile).getFileName().toString(); + } + includeFile(file.toFile(), fixedFile, + archiveOutputStream); + } + if (!hardLinkFiles.isEmpty()) { + Path hardLinkFile = createHardLinkList(truncateLength, hardLinkFiles); + includeFile(hardLinkFile.toFile(), "hardLinkFile", + archiveOutputStream); + } + } + private Path createHardLinkList(int truncateLength, Map hardLinkFiles) @@ -239,11 +252,11 @@ private Path createHardLinkList(int truncateLength, Path data = Files.createTempFile("hardLinkData", "txt"); StringBuilder sb = new StringBuilder(); for (Map.Entry entry : hardLinkFiles.entrySet()) { - String fixedFile = fixFile(truncateLength, entry.getValue()); + String fixedFile = fixFileName(truncateLength, entry.getValue()); if (fixedFile.startsWith("/db.checkpoints")) { fixedFile = Paths.get(fixedFile).getFileName().toString(); } - sb.append(fixFile(truncateLength, entry.getKey())) + sb.append(fixFileName(truncateLength, entry.getKey())) .append("\t") .append(fixedFile) .append("\n"); @@ -252,7 +265,7 @@ private Path createHardLinkList(int truncateLength, return data; } - static String fixFile(int truncateLength, Path file) { + static String fixFileName(int truncateLength, Path file) { return file.toString().substring(truncateLength); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 976da353732..1fe091187fc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -300,7 +300,14 @@ protected OmMetadataManagerImpl() { this.omEpoch = 0; } - // metadata constructor for checkpoints + /** + * metadata constructor for checkpoints + * + * @param conf - ozone conf + * @param dir - checkpoint parent directory + * @param name - checkpoint directory name + * @throws IOException + */ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) throws IOException { lock = new OmReadOnlyLock(); From 8d8fa65edde16e090e8b50396af310fad7a57e28 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 10 Oct 2022 15:49:15 -0700 Subject: [PATCH 015/120] cleanup starting slash --- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 11 ++++++----- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 12 ++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index fe06b54cb29..0829c0d195f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -76,6 +76,7 @@ import org.junit.rules.Timeout; import org.mockito.Matchers; +import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.OM_HARDLINK_FILE; import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.fixFileName; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; @@ -315,12 +316,12 @@ public void testWriteArchiveToStream() // Untar the file into a temp folder to be examined String testDirName = folder.newFolder().getAbsolutePath(); - int testDirLength = testDirName.length(); + int testDirLength = testDirName.length() + 1; FileUtil.unTar(tempFile, new File(testDirName)); Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); - int metaDirLength = metaDir.toString().length(); + int metaDirLength = metaDir.toString().length() + 1; Path finalCheckpointLocation = Paths.get(testDirName); @@ -329,13 +330,13 @@ public void testWriteArchiveToStream() try (Stream files = Files.list(checkpointLocation)) { for (Path file : files.collect(Collectors.toList())) { initialCheckpointSet.add( - fixFileName(checkpointLocation.toString().length(), file)); + fixFileName(checkpointLocation.toString().length() + 1, file)); } } Set finalCheckpointSet = new HashSet<>(); try (Stream files = Files.list(finalCheckpointLocation)) { for (Path file : files.collect(Collectors.toList())) { - if (file.getFileName().toString().equals("db.snapshots")) { + if (file.getFileName().toString().equals(OM_SNAPSHOT_DIR)) { continue; } finalCheckpointSet.add(fixFileName(testDirLength, file)); @@ -343,7 +344,7 @@ public void testWriteArchiveToStream() } // Confirm hardLinkFile exists in checkpoint dir - String hlPath = "/hardLinkFile"; + String hlPath = OM_HARDLINK_FILE; Assert.assertTrue(finalCheckpointSet.contains(hlPath)); finalCheckpointSet.remove(hlPath); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index d381d390bf0..b865704a497 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -56,6 +56,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; @@ -80,6 +81,8 @@ public class OMDBCheckpointServlet extends DBCheckpointServlet { private static final long serialVersionUID = 1L; private static final int MAX_SECONDS_TO_WAIT = 10; + public static final String OM_HARDLINK_FILE = "hardLinkFile"; + @Override public void init() throws ServletException { @@ -228,11 +231,12 @@ private void writeFilesToArchive(HashMap copyFiles, .getConfiguration(); File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); - int truncateLength = metaDirPath.toString().length(); + int truncateLength = metaDirPath.toString().length() + 1; for (Path file : copyFiles.values()) { String fixedFile = fixFileName(truncateLength, file); - if (fixedFile.startsWith("/db.checkpoints")) { + if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { + // checkpoint files go to root of tarball fixedFile = Paths.get(fixedFile).getFileName().toString(); } includeFile(file.toFile(), fixedFile, @@ -240,7 +244,7 @@ private void writeFilesToArchive(HashMap copyFiles, } if (!hardLinkFiles.isEmpty()) { Path hardLinkFile = createHardLinkList(truncateLength, hardLinkFiles); - includeFile(hardLinkFile.toFile(), "hardLinkFile", + includeFile(hardLinkFile.toFile(), OM_HARDLINK_FILE, archiveOutputStream); } } @@ -253,7 +257,7 @@ private Path createHardLinkList(int truncateLength, StringBuilder sb = new StringBuilder(); for (Map.Entry entry : hardLinkFiles.entrySet()) { String fixedFile = fixFileName(truncateLength, entry.getValue()); - if (fixedFile.startsWith("/db.checkpoints")) { + if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { fixedFile = Paths.get(fixedFile).getFileName().toString(); } sb.append(fixFileName(truncateLength, entry.getKey())) From 879695fdaf618b151113875e343fccf3b6e7495d Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 10 Oct 2022 16:28:10 -0700 Subject: [PATCH 016/120] cleanup --- .../ozone/om/TestOMDbCheckpointServlet.java | 115 ++++++++++-------- 1 file changed, 62 insertions(+), 53 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 0829c0d195f..0076139c7dc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -66,6 +66,7 @@ import org.apache.ozone.test.GenericTestUtils; +import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Assert; @@ -99,6 +100,9 @@ public class TestOMDbCheckpointServlet { private HttpServletResponse responseMock = null; private OMDBCheckpointServlet omDbCheckpointServletMock = null; private File metaDir; + private String snapshotDirName; + private String snapshotDirName2; + private DBCheckpoint dbCheckpoint; @Rule public Timeout timeout = Timeout.seconds(240); @@ -280,35 +284,7 @@ public void testSpnegoEnabled() throws Exception { @Test public void testWriteArchiveToStream() throws Exception { - setupCluster(); - metaDir = OMStorage.getOmDbDir(conf); - - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), - "content"); - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), - "content"); - - // this sleep can be removed after this is fixed: - // https://issues.apache.org/jira/browse/HDDS-7279 - Thread.sleep(2000); - String snapshotDirName = - createSnapshot(bucket.getVolumeName(), bucket.getName()); - String snapshotDirName2 = - createSnapshot(bucket.getVolumeName(), bucket.getName()); - - - // create dummy link from one snapshot dir to the other - // to confirm that links are recognized even if - // they are not in the checkpoint directory - Path dummyFile = Paths.get(snapshotDirName, "dummyFile"); - Path dummyLink = Paths.get(snapshotDirName2, "dummyFile"); - Files.write(dummyFile, "dummyData".getBytes(StandardCharsets.UTF_8)); - Files.createLink(dummyLink, dummyFile); - - DBCheckpoint dbCheckpoint = cluster.getOzoneManager() - .getMetadataManager().getStore() - .getCheckpoint(true); + prepArchiveData(); FileOutputStream fileOutputStream = new FileOutputStream(tempFile); omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, @@ -325,31 +301,16 @@ public void testWriteArchiveToStream() Path finalCheckpointLocation = Paths.get(testDirName); - // Confirm the checkpoint directories match - Set initialCheckpointSet = new HashSet<>(); - try (Stream files = Files.list(checkpointLocation)) { - for (Path file : files.collect(Collectors.toList())) { - initialCheckpointSet.add( - fixFileName(checkpointLocation.toString().length() + 1, file)); - } - } - Set finalCheckpointSet = new HashSet<>(); - try (Stream files = Files.list(finalCheckpointLocation)) { - for (Path file : files.collect(Collectors.toList())) { - if (file.getFileName().toString().equals(OM_SNAPSHOT_DIR)) { - continue; - } - finalCheckpointSet.add(fixFileName(testDirLength, file)); - } - } + // Confirm the checkpoint directories match, (after remove extras) + Set initialCheckpointSet = getFiles(checkpointLocation, + checkpointLocation.toString().length() + 1); + Set finalCheckpointSet = getFiles(finalCheckpointLocation, + testDirLength); + finalCheckpointSet.remove(OM_SNAPSHOT_DIR); // Confirm hardLinkFile exists in checkpoint dir - String hlPath = OM_HARDLINK_FILE; - Assert.assertTrue(finalCheckpointSet.contains(hlPath)); - - finalCheckpointSet.remove(hlPath); - - + Assert.assertTrue(finalCheckpointSet.contains(OM_HARDLINK_FILE)); + finalCheckpointSet.remove(OM_HARDLINK_FILE); Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); String shortSnapshotLocation = @@ -391,7 +352,7 @@ public void testWriteArchiveToStream() Assert.assertTrue("snapshot manifest found", foundManifest); // check each line in the hard link file - Stream lines = Files.lines(Paths.get(testDirName, hlPath)); + Stream lines = Files.lines(Paths.get(testDirName, OM_HARDLINK_FILE)); boolean linesFound = false; boolean dummyLinkFound = false; for (String line: lines.collect(Collectors.toList())) { @@ -412,6 +373,20 @@ public void testWriteArchiveToStream() initialSnapshotSet, finalSnapshotSet); } + + @NotNull + private Set getFiles(Path path, int truncateLength) + throws IOException { + Set fileSet = new HashSet<>(); + try (Stream files = Files.list(path)) { + for (Path file : files.collect(Collectors.toList())) { + fileSet.add( + fixFileName(truncateLength, file)); + } + } + return fileSet; + } + private boolean checkDummyFile(String dir0, String dir1, String line) { String[] files = line.split("\t"); if (!files[0].startsWith(dir0) && !files[1].startsWith(dir0)) { @@ -462,4 +437,38 @@ private String createSnapshot(String vname, String bname) return snapshotDirName; } + private void prepArchiveData() { + + setupCluster(); + metaDir = OMStorage.getOmDbDir(conf); + + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + "content"); + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + "content"); + + // this sleep can be removed after this is fixed: + // https://issues.apache.org/jira/browse/HDDS-7279 + Thread.sleep(2000); + snapshotDirName = + createSnapshot(bucket.getVolumeName(), bucket.getName()); + snapshotDirName2 = + createSnapshot(bucket.getVolumeName(), bucket.getName()); + + + // create dummy link from one snapshot dir to the other + // to confirm that links are recognized even if + // they are not in the checkpoint directory + Path dummyFile = Paths.get(snapshotDirName, "dummyFile"); + Path dummyLink = Paths.get(snapshotDirName2, "dummyFile"); + Files.write(dummyFile, "dummyData".getBytes(StandardCharsets.UTF_8)); + Files.createLink(dummyLink, dummyFile); + + dbCheckpoint = cluster.getOzoneManager() + .getMetadataManager().getStore() + .getCheckpoint(true); + + + } } From 354b280ed0319cda9be7cb88c3a878ab12e0718f Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 10 Oct 2022 16:46:02 -0700 Subject: [PATCH 017/120] cleanup --- .../ozone/om/TestOMDbCheckpointServlet.java | 55 ++++++------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 0076139c7dc..971ef5c6512 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -308,8 +308,8 @@ public void testWriteArchiveToStream() testDirLength); finalCheckpointSet.remove(OM_SNAPSHOT_DIR); - // Confirm hardLinkFile exists in checkpoint dir - Assert.assertTrue(finalCheckpointSet.contains(OM_HARDLINK_FILE)); + Assert.assertTrue("hardlink file exists in checkpoint dir", + finalCheckpointSet.contains(OM_HARDLINK_FILE)); finalCheckpointSet.remove(OM_HARDLINK_FILE); Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); @@ -317,46 +317,26 @@ public void testWriteArchiveToStream() fixFileName(metaDirLength, Paths.get(snapshotDirName)); String shortSnapshotLocation2 = fixFileName(metaDirLength, Paths.get(snapshotDirName2)); - Path finalSnapshotLocation = Paths.get(testDirName, shortSnapshotLocation); - // Confirm snapshot current file exists and is not a hard link - Assert.assertTrue(Paths - .get(finalSnapshotLocation.toString(), - "CURRENT").toFile().exists()); - - // get initial snapshot files - Set initialSnapshotSet = new HashSet<>(); - try (Stream files = Files.list(Paths.get(snapshotDirName))) { - for (Path file : files.collect(Collectors.toList())) { - if (!file.getFileName().toString().equals("dummyFile")) { - initialSnapshotSet.add(fixFileName(metaDirLength, file)); - } - } - } + Assert.assertTrue("CURRENT file exists", + Paths.get(finalSnapshotLocation.toString(), + "CURRENT").toFile().exists()); - // get final snapshot files - Set finalSnapshotSet = new HashSet<>(); - boolean foundManifest = false; - try (Stream files = Files.list(finalSnapshotLocation)) { - for (Path file : files.collect(Collectors.toList())) { - if (file.toString().contains("MANIFEST")) { - foundManifest = true; - } - if (!file.getFileName().toString().equals("dummyFile")) { - finalSnapshotSet.add(fixFileName(testDirLength, file)); - } - } - } - Assert.assertTrue("snapshot manifest found", foundManifest); + Set initialSnapshotSet = + getFiles(Paths.get(snapshotDirName), metaDirLength); + Set finalSnapshotSet = + getFiles(finalSnapshotLocation, testDirLength); + Assert.assertTrue("snapshot manifest found", + finalSnapshotSet.stream().anyMatch(s->s.contains("MANIFEST"))); // check each line in the hard link file Stream lines = Files.lines(Paths.get(testDirName, OM_HARDLINK_FILE)); - boolean linesFound = false; boolean dummyLinkFound = false; for (String line: lines.collect(Collectors.toList())) { - linesFound = true; + Assert.assertFalse("CURRENT file is not a hard link", + line.contains("CURRENT")); if (line.contains("dummyFile")) { dummyLinkFound = true; Assert.assertTrue(checkDummyFile(shortSnapshotLocation,shortSnapshotLocation2, line)); @@ -367,11 +347,9 @@ public void testWriteArchiveToStream() } } } - Assert.assertTrue("hard link file not empty", linesFound); Assert.assertTrue("dummy link found", dummyLinkFound); Assert.assertEquals("found expected snapshot files", initialSnapshotSet, finalSnapshotSet); - } @NotNull @@ -380,8 +358,9 @@ private Set getFiles(Path path, int truncateLength) Set fileSet = new HashSet<>(); try (Stream files = Files.list(path)) { for (Path file : files.collect(Collectors.toList())) { - fileSet.add( - fixFileName(truncateLength, file)); + if (!file.getFileName().toString().equals("dummyFile")) { + fileSet.add(fixFileName(truncateLength, file)); + } } } return fileSet; @@ -437,7 +416,7 @@ private String createSnapshot(String vname, String bname) return snapshotDirName; } - private void prepArchiveData() { + private void prepArchiveData() throws Exception { setupCluster(); metaDir = OMStorage.getOmDbDir(conf); From 045b701c4d894d3920df76431452a5e0ed762618 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 10 Oct 2022 17:08:32 -0700 Subject: [PATCH 018/120] cleanup --- .../ozone/om/TestOMDbCheckpointServlet.java | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 971ef5c6512..929919a529f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -282,7 +282,7 @@ public void testSpnegoEnabled() throws Exception { } @Test - public void testWriteArchiveToStream() + public void testWriteDbDataToStream() throws Exception { prepArchiveData(); @@ -339,9 +339,9 @@ public void testWriteArchiveToStream() line.contains("CURRENT")); if (line.contains("dummyFile")) { dummyLinkFound = true; - Assert.assertTrue(checkDummyFile(shortSnapshotLocation,shortSnapshotLocation2, line)); + checkDummyFile(shortSnapshotLocation,shortSnapshotLocation2, line); } else { - Assert.assertTrue(checkLine(shortSnapshotLocation, shortSnapshotLocation2, line)); + checkLine(shortSnapshotLocation, shortSnapshotLocation2, line); if (line.startsWith(shortSnapshotLocation)) { finalSnapshotSet.add(line.split("\t")[0]); } @@ -366,37 +366,38 @@ private Set getFiles(Path path, int truncateLength) return fileSet; } - private boolean checkDummyFile(String dir0, String dir1, String line) { + // tests to see that dummy entry in hardlink file looks something like: + // "dir1/dummyFile dir2/dummyFile" + private void checkDummyFile(String dir0, String dir1, String line) { String[] files = line.split("\t"); - if (!files[0].startsWith(dir0) && !files[1].startsWith(dir0)) { - return false; - } - if (!files[0].startsWith(dir1) && !files[1].startsWith(dir1)) { - return false; - } + Assert.assertTrue("dummy entry contains first directory", + files[0].startsWith(dir0) || files[1].startsWith(dir0)); + Assert.assertTrue("dummy entry contains second directory", + files[0].startsWith(dir1) || files[1].startsWith(dir1)); Path path0 = Paths.get(files[0]); Path path1 = Paths.get(files[1]); - if (path0.getParent().equals(path1.getParent())) { - return false; - } - return path0.getFileName().equals(path1.getFileName()); + Assert.assertNotEquals("dummy entry parent directories differ", + path0.getParent(), path1.getParent()); + Assert.assertTrue("dummy entries contains dummyFile name", + path0.getFileName().toString().equals("dummyFile") && + path1.getFileName().toString().equals("dummyFile")); } - private boolean checkLine(String shortSnapshotLocation, + + // tests line in hard link file looks something like: + // "dir1/x.sst x.sst" + private void checkLine(String shortSnapshotLocation, String shortSnapshotLocation2, String line) { String[] files = line.split("\t"); - if (!files[0].startsWith(shortSnapshotLocation) && - !files[0].startsWith(shortSnapshotLocation2)) { - return false; - } + Assert.assertTrue("hl entry starts with valid snapshot dir", + files[0].startsWith(shortSnapshotLocation) || + files[0].startsWith(shortSnapshotLocation2)); - String file0 = files[0].substring(shortSnapshotLocation.length()); + String file0 = files[0].substring(shortSnapshotLocation.length() + 1); String file1 = files[1]; - if (Paths.get(file0).getNameCount() > 1) { - return false; - } - return file0.equals("/" + file1); + Assert.assertEquals("hl filenames are the same", file0, file1); } + private String createSnapshot(String vname, String bname) throws IOException, InterruptedException, TimeoutException { final OzoneManager om = cluster.getOzoneManager(); @@ -447,7 +448,5 @@ private void prepArchiveData() throws Exception { dbCheckpoint = cluster.getOzoneManager() .getMetadataManager().getStore() .getCheckpoint(true); - - } } From 407d9b1380a18a75b3590f510ae4cdc8f96984af Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 11 Oct 2022 14:38:34 -0700 Subject: [PATCH 019/120] added duration --- .../ozone/om/OMDBCheckpointServlet.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index b865704a497..f6ac288f3ed 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -47,6 +47,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; @@ -79,7 +80,7 @@ public class OMDBCheckpointServlet extends DBCheckpointServlet { private static final Logger LOG = LoggerFactory.getLogger(OMDBCheckpointServlet.class); private static final long serialVersionUID = 1L; - private static final int MAX_SECONDS_TO_WAIT = 10; + private static final String DURATION_TO_WAIT_FOR_DIRECTORY = "PT10S"; public static final String OM_HARDLINK_FILE = "hardLinkFile"; @@ -164,15 +165,19 @@ private List getSnapshotDirs(DBCheckpoint checkpoint) private void waitForDirToExist(Path dir) throws IOException, InterruptedException { - int count = 0; - while (!dir.toFile().exists() && count < MAX_SECONDS_TO_WAIT) { - Thread.sleep(1000); - count++; + long endTime = System.currentTimeMillis() + + Duration.parse(DURATION_TO_WAIT_FOR_DIRECTORY).toMillis(); + while (!dir.toFile().exists()) { + Thread.sleep(100); + if (System.currentTimeMillis() > endTime) { + break; + } } - if (count == MAX_SECONDS_TO_WAIT) { + if (System.currentTimeMillis() > endTime) { throw new IOException("snapshot dir doesn't exist: " + dir); } } + private void processDir(Path dir, Map copyFiles, Map hardLinkFiles) throws IOException, InterruptedException { @@ -253,7 +258,7 @@ private void writeFilesToArchive(HashMap copyFiles, private Path createHardLinkList(int truncateLength, Map hardLinkFiles) throws IOException { - Path data = Files.createTempFile("hardLinkData", "txt"); + Path data = Files.createTempFile("data", "txt"); StringBuilder sb = new StringBuilder(); for (Map.Entry entry : hardLinkFiles.entrySet()) { String fixedFile = fixFileName(truncateLength, entry.getValue()); From 31a048cfc5d322343027db568642c3ec18a7384f Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 11 Oct 2022 15:29:45 -0700 Subject: [PATCH 020/120] getSnapshotPath() --- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 6 +++--- .../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 13 ++++--------- .../hadoop/ozone/om/TestOmSnapshotFileSystem.java | 11 +++-------- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 5 ++--- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 11 +++++++++++ .../snapshot/TestOMSnapshotCreateResponse.java | 8 ++++---- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 929919a529f..0b238a71aa4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -79,6 +79,7 @@ import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.OM_HARDLINK_FILE; import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.fixFileName; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doNothing; @@ -409,9 +410,8 @@ private String createSnapshot(String vname, String bname) SnapshotInfo snapshotInfo = om .getMetadataManager().getSnapshotInfoTable() .get(SnapshotInfo.getTableKey(vname, bname, snapshotName)); - String snapshotDirName = metaDir + OM_KEY_PREFIX + - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + OM_DB_NAME + - snapshotInfo.getCheckpointDirName() + OM_KEY_PREFIX; + String snapshotDirName = getSnapshotPath(conf, snapshotInfo) + + OM_KEY_PREFIX; GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), 100, 2000); return snapshotDirName; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java index 8705f9356d1..c69cf342752 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java @@ -61,9 +61,7 @@ import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.client.ReplicationType.RATIS; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE; @@ -85,7 +83,7 @@ public class TestOmSnapshot { private static BucketLayout bucketLayout = BucketLayout.LEGACY; private static boolean enabledFileSystemPaths; private static ObjectStore store; - private static File metaDir; + private static OzoneConfiguration conf; private static OzoneManager ozoneManager; @Rule @@ -126,7 +124,7 @@ private static void setConfig(BucketLayout newBucketLayout, * */ private void init() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); + conf = new OzoneConfiguration(); String clusterId = UUID.randomUUID().toString(); String scmId = UUID.randomUUID().toString(); String omId = UUID.randomUUID().toString(); @@ -152,7 +150,6 @@ private void init() throws Exception { // stop the deletion services so that keys can still be read keyManager.stop(); - metaDir = OMStorage.getOmDbDir(conf); } @@ -440,9 +437,7 @@ private String createSnapshot(String vname, String bname) SnapshotInfo snapshotInfo = ozoneManager .getMetadataManager().getSnapshotInfoTable() .get(SnapshotInfo.getTableKey(vname, bname, snapshotName)); - String snapshotDirName = metaDir + OM_KEY_PREFIX + - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + OM_DB_NAME + - snapshotInfo.getCheckpointDirName() + OM_KEY_PREFIX + "CURRENT"; + String snapshotDirName = getSnapshotPath(conf, snapshotInfo); GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), 1000, 120000); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java index 36424e4a39e..d7d0a4a090e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java @@ -73,10 +73,9 @@ import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; @@ -95,8 +94,7 @@ public class TestOmSnapshotFileSystem { private static OzoneManagerProtocol writeClient; private static BucketLayout bucketLayout; private static boolean enabledFileSystemPaths; - private static File metaDir; - private static OzoneManager ozoneManager; + private static OzoneManager ozoneManager; private static String keyPrefix; private static final Logger LOG = @@ -169,7 +167,6 @@ private void init() throws Exception { ObjectStore objectStore = client.getObjectStore(); writeClient = objectStore.getClientProxy().getOzoneManagerClient(); ozoneManager = cluster.getOzoneManager(); - metaDir = OMStorage.getOmDbDir(conf); // stop the deletion services so that keys can still be read KeyManagerImpl keyManager = (KeyManagerImpl) ozoneManager.getKeyManager(); @@ -570,9 +567,7 @@ private String createSnapshot() SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager() .getSnapshotInfoTable() .get(SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName)); - String snapshotDirName = metaDir + OM_KEY_PREFIX + - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + OM_DB_NAME + - snapshotInfo.getCheckpointDirName() + OM_KEY_PREFIX + "CURRENT"; + String snapshotDirName = getSnapshotPath(conf, snapshotInfo); GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), 1000, 120000); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index f6ac288f3ed..9ffd08bce33 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -61,6 +61,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; /** * Provides the current checkpoint Snapshot of the OM DB. (tar.gz) @@ -154,9 +155,7 @@ private List getSnapshotDirs(DBCheckpoint checkpoint) // add each entries directory to the list while (iterator.hasNext()) { Table.KeyValue entry = iterator.next(); - Path path = Paths.get(OMStorage.getOmDbDir(conf) + - OM_KEY_PREFIX + OM_SNAPSHOT_DIR + OM_KEY_PREFIX + - OM_DB_NAME + entry.getValue().getCheckpointDirName()); + Path path = Paths.get(getSnapshotPath(conf, entry.getValue())); list.add(path); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 9691fb366d4..0082a9f75b5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -40,9 +40,13 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.concurrent.ExecutionException; +import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; @@ -220,6 +224,13 @@ public static String getSnapshotPrefix(String snapshotName) { snapshotName + OM_KEY_PREFIX; } + public static String getSnapshotPath(OzoneConfiguration conf, + SnapshotInfo snapshotInfo) { + return OMStorage.getOmDbDir(conf) + + OM_KEY_PREFIX + OM_SNAPSHOT_DIR + OM_KEY_PREFIX + + OM_DB_NAME + snapshotInfo.getCheckpointDirName(); + } + public static boolean isSnapshotKey(String[] keyParts) { return (keyParts.length > 1) && (keyParts[0].compareTo(OM_SNAPSHOT_INDICATOR) == 0); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java index 755fed158b1..3895eb9f9e0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java @@ -44,6 +44,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; /** @@ -57,9 +58,10 @@ public class TestOMSnapshotCreateResponse { private OMMetadataManager omMetadataManager; private BatchOperation batchOperation; private String fsPath; + private OzoneConfiguration ozoneConfiguration; @Before public void setup() throws Exception { - OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration = new OzoneConfiguration(); fsPath = folder.newFolder().getAbsolutePath(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); @@ -100,9 +102,7 @@ public void testAddToDBBatch() throws Exception { omMetadataManager.getStore().commitBatchOperation(batchOperation); // Confirm snapshot directory was created - String snapshotDir = fsPath + OM_KEY_PREFIX + - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + OM_DB_NAME + - snapshotInfo.getCheckpointDirName(); + String snapshotDir = getSnapshotPath(ozoneConfiguration, snapshotInfo); Assert.assertTrue((new File(snapshotDir)).exists()); // Confirm table has 1 entry From 16968e6fba0745d34033bb60b1b981927ba97af1 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 11 Oct 2022 15:45:38 -0700 Subject: [PATCH 021/120] checks dummy file count --- .../ozone/om/TestOMDbCheckpointServlet.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 0b238a71aa4..eb3dbfa3fc4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -58,7 +58,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH; @@ -340,7 +339,8 @@ public void testWriteDbDataToStream() line.contains("CURRENT")); if (line.contains("dummyFile")) { dummyLinkFound = true; - checkDummyFile(shortSnapshotLocation,shortSnapshotLocation2, line); + checkDummyFile(shortSnapshotLocation,shortSnapshotLocation2, line, + testDirName); } else { checkLine(shortSnapshotLocation, shortSnapshotLocation2, line); if (line.startsWith(shortSnapshotLocation)) { @@ -369,7 +369,8 @@ private Set getFiles(Path path, int truncateLength) // tests to see that dummy entry in hardlink file looks something like: // "dir1/dummyFile dir2/dummyFile" - private void checkDummyFile(String dir0, String dir1, String line) { + private void checkDummyFile(String dir0, String dir1, String line, + String testDirName) { String[] files = line.split("\t"); Assert.assertTrue("dummy entry contains first directory", files[0].startsWith(dir0) || files[1].startsWith(dir0)); @@ -382,6 +383,15 @@ private void checkDummyFile(String dir0, String dir1, String line) { Assert.assertTrue("dummy entries contains dummyFile name", path0.getFileName().toString().equals("dummyFile") && path1.getFileName().toString().equals("dummyFile")); + int dummyFileCount = 0; + if (Paths.get(testDirName, dir0, "dummyFile").toFile().exists()) { + dummyFileCount++; + } + if (Paths.get(testDirName, dir1, "dummyFile").toFile().exists()) { + dummyFileCount++; + } + Assert.assertEquals("exactly one dummy file exists in the output dir", + dummyFileCount, 1); } // tests line in hard link file looks something like: From 713dcad9fd3b10b74059a148a3f344bcf5a3c039 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 11 Oct 2022 16:04:36 -0700 Subject: [PATCH 022/120] checkstyle --- .../ozone/om/TestOMDbCheckpointServlet.java | 24 ++++++++++--------- .../ozone/om/TestOmSnapshotFileSystem.java | 2 +- .../ozone/om/OMDBCheckpointServlet.java | 14 +++++------ .../ozone/om/OmMetadataManagerImpl.java | 2 +- .../TestOMSnapshotCreateResponse.java | 3 --- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index eb3dbfa3fc4..cf1b079adb0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -65,7 +65,6 @@ import org.apache.ozone.test.GenericTestUtils; -import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Assert; @@ -286,9 +285,10 @@ public void testWriteDbDataToStream() throws Exception { prepArchiveData(); - FileOutputStream fileOutputStream = new FileOutputStream(tempFile); - omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, - fileOutputStream); + try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { + omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, + fileOutputStream); + } // Untar the file into a temp folder to be examined String testDirName = folder.newFolder().getAbsolutePath(); @@ -329,17 +329,18 @@ public void testWriteDbDataToStream() Set finalSnapshotSet = getFiles(finalSnapshotLocation, testDirLength); Assert.assertTrue("snapshot manifest found", - finalSnapshotSet.stream().anyMatch(s->s.contains("MANIFEST"))); + finalSnapshotSet.stream().anyMatch(s -> s.contains("MANIFEST"))); // check each line in the hard link file - Stream lines = Files.lines(Paths.get(testDirName, OM_HARDLINK_FILE)); + Stream lines = Files.lines(Paths.get(testDirName, + OM_HARDLINK_FILE)); boolean dummyLinkFound = false; for (String line: lines.collect(Collectors.toList())) { Assert.assertFalse("CURRENT file is not a hard link", line.contains("CURRENT")); if (line.contains("dummyFile")) { dummyLinkFound = true; - checkDummyFile(shortSnapshotLocation,shortSnapshotLocation2, line, + checkDummyFile(shortSnapshotLocation, shortSnapshotLocation2, line, testDirName); } else { checkLine(shortSnapshotLocation, shortSnapshotLocation2, line); @@ -353,14 +354,15 @@ public void testWriteDbDataToStream() initialSnapshotSet, finalSnapshotSet); } - @NotNull private Set getFiles(Path path, int truncateLength) throws IOException { Set fileSet = new HashSet<>(); try (Stream files = Files.list(path)) { for (Path file : files.collect(Collectors.toList())) { - if (!file.getFileName().toString().equals("dummyFile")) { + if (file != null) { + if (!file.getFileName().toString().equals("dummyFile")) { fileSet.add(fixFileName(truncateLength, file)); + } } } } @@ -420,9 +422,9 @@ private String createSnapshot(String vname, String bname) SnapshotInfo snapshotInfo = om .getMetadataManager().getSnapshotInfoTable() .get(SnapshotInfo.getTableKey(vname, bname, snapshotName)); - String snapshotDirName = getSnapshotPath(conf, snapshotInfo) + String snapshotPath = getSnapshotPath(conf, snapshotInfo) + OM_KEY_PREFIX; - GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), + GenericTestUtils.waitFor(() -> new File(snapshotPath).exists(), 100, 2000); return snapshotDirName; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java index d7d0a4a090e..ae33ca9d10e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java @@ -94,7 +94,7 @@ public class TestOmSnapshotFileSystem { private static OzoneManagerProtocol writeClient; private static BucketLayout bucketLayout; private static boolean enabledFileSystemPaths; - private static OzoneManager ozoneManager; + private static OzoneManager ozoneManager; private static String keyPrefix; private static final Logger LOG = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 9ffd08bce33..d0a7d3ced32 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -58,9 +58,6 @@ import java.util.stream.Stream; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; /** @@ -124,7 +121,7 @@ private void getFilesForArchive(DBCheckpoint checkpoint, Path dir = checkpoint.getCheckpointLocation(); try (Stream files = Files.list(dir)) { - for(Path file : files.collect(Collectors.toList())) { + for (Path file : files.collect(Collectors.toList())) { // get the inode Object key = Files.readAttributes( file, BasicFileAttributes.class).fileKey(); @@ -148,7 +145,7 @@ private List getSnapshotDirs(DBCheckpoint checkpoint) OmMetadataManagerImpl checkpointMetadataManager = OmMetadataManagerImpl.createCheckpointMetadataManager( conf, checkpoint); - try(TableIterator> + try (TableIterator> iterator = checkpointMetadataManager .getSnapshotInfoTable().iterator()) { @@ -177,12 +174,12 @@ private void waitForDirToExist(Path dir) } } - private void processDir(Path dir, Map copyFiles, + private void processDir(Path dir, Map copyFiles, Map hardLinkFiles) throws IOException, InterruptedException { waitForDirToExist(dir); try (Stream files = Files.list(dir)) { - for(Path file : files.collect(Collectors.toList())) { + for (Path file : files.collect(Collectors.toList())) { // get the inode Object key = Files.readAttributes( file, BasicFileAttributes.class).fileKey(); @@ -198,7 +195,8 @@ private void processDir(Path dir, Map copyFiles, } @Override - public void writeDbDataToStream(DBCheckpoint checkpoint, OutputStream destination) + public void writeDbDataToStream(DBCheckpoint checkpoint, + OutputStream destination) throws IOException, InterruptedException, CompressorException { // Map of inodes to path HashMap copyFiles = new HashMap<>(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 1fe091187fc..7fe896aa5f8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -301,7 +301,7 @@ protected OmMetadataManagerImpl() { } /** - * metadata constructor for checkpoints + * metadata constructor for checkpoints. * * @param conf - ozone conf * @param dir - checkpoint parent directory diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java index 3895eb9f9e0..21ee4ae79b7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java @@ -41,9 +41,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .OMResponse; import org.apache.hadoop.hdds.utils.db.BatchOperation; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; From 25cdbd1fb48ed93b68f5af3f75f716f3a246f00d Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 11 Oct 2022 16:11:00 -0700 Subject: [PATCH 023/120] fixed snapshotDirName --- .../org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index cf1b079adb0..9161d2e529f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -426,7 +426,7 @@ private String createSnapshot(String vname, String bname) + OM_KEY_PREFIX; GenericTestUtils.waitFor(() -> new File(snapshotPath).exists(), 100, 2000); - return snapshotDirName; + return snapshotPath; } private void prepArchiveData() throws Exception { From e329e8d3325df2de16c79a3deb953c424806d05a Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 11 Oct 2022 16:17:09 -0700 Subject: [PATCH 024/120] cleanup --- .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 5 ++--- .../om/response/snapshot/TestOMSnapshotCreateResponse.java | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 0082a9f75b5..2cfdd2cadbb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -40,8 +40,7 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.util.Objects; import java.util.concurrent.ExecutionException; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; @@ -120,7 +119,7 @@ public OmSnapshot load(@Nonnull String snapshotTableKey) = notification -> { try { // close snapshot's rocksdb on eviction - notification.getValue().close(); + Objects.requireNonNull(notification.getValue()).close(); } catch (IOException e) { LOG.error("Failed to close snapshot: {} {}", notification.getKey(), e); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java index 21ee4ae79b7..d93619b5077 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java @@ -54,12 +54,11 @@ public class TestOMSnapshotCreateResponse { private OMMetadataManager omMetadataManager; private BatchOperation batchOperation; - private String fsPath; private OzoneConfiguration ozoneConfiguration; @Before public void setup() throws Exception { ozoneConfiguration = new OzoneConfiguration(); - fsPath = folder.newFolder().getAbsolutePath(); + String fsPath = folder.newFolder().getAbsolutePath(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration); From 6975c1a6517912a8dced3d3d57cde549e1522660 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 2 Nov 2022 23:11:46 -0700 Subject: [PATCH 025/120] follower installs snapshot tarball --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 75 +++++++++++++++++++ .../apache/hadoop/ozone/om/OzoneManager.java | 59 ++++++++++++++- 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 1c787a426f5..29c799d9e5e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -41,6 +41,8 @@ import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.hdds.ExitManager; @@ -537,6 +539,79 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { Assert.assertTrue(logCapture.getOutput().contains(msg)); } + @Test + public void testOmSnapshots() throws Exception { + // Get the leader OM + String leaderOMNodeId = OmFailoverProxyUtil + .getFailoverProxyProvider(objectStore.getClientProxy()) + .getCurrentProxyOMNodeId(); + + OzoneManager leaderOM = cluster.getOzoneManager(leaderOMNodeId); + OzoneManagerRatisServer leaderRatisServer = leaderOM.getOmRatisServer(); + + // Find the inactive OM + String followerNodeId = leaderOM.getPeerNodes().get(0).getNodeId(); + if (cluster.isOMActive(followerNodeId)) { + followerNodeId = leaderOM.getPeerNodes().get(1).getNodeId(); + } + OzoneManager followerOM = cluster.getOzoneManager(followerNodeId); + + // Do some transactions so that the log index increases + List keys = writeKeysToIncreaseLogIndex(leaderRatisServer, 200); + + objectStore.createSnapshot(volumeName, bucketName, "snap1"); + + // Get the latest db checkpoint from the leader OM. + TransactionInfo transactionInfo = + TransactionInfo.readTransactionInfo(leaderOM.getMetadataManager()); + TermIndex leaderOMTermIndex = + TermIndex.valueOf(transactionInfo.getTerm(), + transactionInfo.getTransactionIndex()); + long leaderOMSnapshotIndex = leaderOMTermIndex.getIndex(); + long leaderOMSnapshotTermIndex = leaderOMTermIndex.getTerm(); + + + + // Start the inactive OM. Checkpoint installation will happen spontaneously. + cluster.startInactiveOM(followerNodeId); + GenericTestUtils.LogCapturer logCapture = + GenericTestUtils.LogCapturer.captureLogs(OzoneManager.LOG); + + // The recently started OM should be lagging behind the leader OM. + // Wait & for follower to update transactions to leader snapshot index. + // Timeout error if follower does not load update within 3s + GenericTestUtils.waitFor(() -> { + return followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex() + >= leaderOMSnapshotIndex - 1; + }, 100, 3000); + + long followerOMLastAppliedIndex = + followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex(); + assertTrue( + followerOMLastAppliedIndex >= leaderOMSnapshotIndex - 1); + + // After the new checkpoint is installed, the follower OM + // lastAppliedIndex must >= the snapshot index of the checkpoint. It + // could be great than snapshot index if there is any conf entry from ratis. + followerOMLastAppliedIndex = followerOM.getOmRatisServer() + .getLastAppliedTermIndex().getIndex(); + assertTrue(followerOMLastAppliedIndex >= leaderOMSnapshotIndex); + assertTrue(followerOM.getOmRatisServer().getLastAppliedTermIndex() + .getTerm() >= leaderOMSnapshotTermIndex); + + // Verify checkpoint installation was happened. + String msg = "Reloaded OM state"; + Assert.assertTrue(logCapture.getOutput().contains(msg)); + + // read back data from the snapshot + OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(".snapshot/snap1/" + keys.get(0)).build(); + OmKeyInfo omKeyInfo = followerOM.lookupKey(omKeyArgs); + Assert.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); + } + private List writeKeysToIncreaseLogIndex( OzoneManagerRatisServer omRatisServer, long targetLogIndex) throws IOException, InterruptedException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 8013e1bd15c..65dbb2e80e8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -29,6 +29,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.security.KeyPair; import java.security.PrivilegedExceptionAction; @@ -48,6 +49,7 @@ import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import com.google.common.base.Optional; import org.apache.hadoop.conf.Configuration; @@ -230,6 +232,8 @@ import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY; import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE; import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; import static org.apache.hadoop.ozone.OzoneConsts.PREPARE_MARKER_KEY; import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT; @@ -3597,10 +3601,20 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, String dbBackupName = OzoneConsts.OM_DB_BACKUP_PREFIX + lastAppliedIndex + "_" + System.currentTimeMillis(); File dbDir = oldDB.getParentFile(); - File dbBackup = new File(dbDir, dbBackupName); - + File dbBackupDir = new File(dbDir, dbBackupName); + if (!dbBackupDir.mkdirs()) { + throw new IOException("Failed to make db backup dir: " + + dbBackupDir.toString()); + } + File dbBackup = new File(dbBackupDir, oldDB.getName()); + File dbSnapshotsDir = new File(dbDir, OM_SNAPSHOT_DIR); + File dbSnapshotsBackup = new File(dbBackupDir, OM_SNAPSHOT_DIR); try { Files.move(oldDB.toPath(), dbBackup.toPath()); + if (dbSnapshotsDir.exists()) { + Files.move(dbSnapshotsDir.toPath(), + dbSnapshotsBackup.toPath()); + } } catch (IOException e) { LOG.error("Failed to create a backup of the current DB. Aborting " + "snapshot installation."); @@ -3617,6 +3631,7 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, // starting up. Files.createFile(markerFile); FileUtils.moveDirectory(checkpointPath, oldDB.toPath()); + moveOmSnapshotData(oldDB.toPath(), dbSnapshotsDir.toPath()); Files.deleteIfExists(markerFile); } catch (IOException e) { LOG.error("Failed to move downloaded DB checkpoint {} to metadata " + @@ -3624,6 +3639,9 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, oldDB.toPath()); try { Files.move(dbBackup.toPath(), oldDB.toPath()); + if (dbSnapshotsBackup.exists()) { + Files.move(dbSnapshotsBackup.toPath(), dbSnapshotsDir.toPath()); + } Files.deleteIfExists(markerFile); } catch (IOException ex) { String errorMsg = "Failed to reset to original DB. OM is in an " + @@ -3632,9 +3650,44 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, } throw e; } - return dbBackup; + return dbBackupDir; } + private void moveOmSnapshotData(Path dbPath, Path dbSnapshotsDir) + throws IOException { + Path incomingSnapshotsDir = Paths.get(dbPath.toString(), OM_SNAPSHOT_DIR); + if (incomingSnapshotsDir.toFile().exists()) { + Files.move(incomingSnapshotsDir, dbSnapshotsDir); + createHardLinks(dbPath); + } + + } + + private void createHardLinks(Path dbPath) throws IOException { + File hardLinkFile = new File(dbPath.toString(), + OMDBCheckpointServlet.OM_HARDLINK_FILE); + if (hardLinkFile.exists()) { + List lines = + Files.lines(hardLinkFile.toPath()).collect(Collectors.toList()); + for (String l : lines) { + String from = l.split("\t")[1]; + String to = l.split("\t")[0]; + Path fixedFrom = fixName(dbPath, from); + Path fixedTo = fixName(dbPath, to); + Files.createLink(fixedTo, fixedFrom); + } + hardLinkFile.delete(); + } else { + throw new RuntimeException("hardlink file not found"); + } + } + + private Path fixName(Path dbPath, String fileName) { + if (fileName.startsWith(OM_SNAPSHOT_DIR)) { + return Paths.get(dbPath.getParent().toString(), fileName); + } + return Paths.get(dbPath.toString(), fileName); + } /** * Re-instantiate MetadataManager with new DB checkpoint. * All the classes which use/ store MetadataManager should also be updated From a1ee8732ee0567b2cd1caca65a2b8dc5c89271c4 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 4 Nov 2022 10:23:07 -0700 Subject: [PATCH 026/120] cleanup --- .../main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 65dbb2e80e8..f80e48a70db 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3677,17 +3677,17 @@ private void createHardLinks(Path dbPath) throws IOException { Files.createLink(fixedTo, fixedFrom); } hardLinkFile.delete(); - } else { - throw new RuntimeException("hardlink file not found"); } } private Path fixName(Path dbPath, String fileName) { - if (fileName.startsWith(OM_SNAPSHOT_DIR)) { + File file = new File(fileName); + if (!file.getName().equals(fileName)) { return Paths.get(dbPath.getParent().toString(), fileName); } return Paths.get(dbPath.toString(), fileName); } + /** * Re-instantiate MetadataManager with new DB checkpoint. * All the classes which use/ store MetadataManager should also be updated From e66271040f977123a8fd9ca208d2b5e10cf42b09 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 4 Nov 2022 13:14:50 -0700 Subject: [PATCH 027/120] refactored hardlink handling --- .../ozone/om/TestOMDbCheckpointServlet.java | 2 +- .../ozone/om/OMDBCheckpointServlet.java | 27 +-------- .../hadoop/ozone/om/OmSnapshotManager.java | 58 +++++++++++++++++++ .../apache/hadoop/ozone/om/OzoneManager.java | 27 +-------- 4 files changed, 62 insertions(+), 52 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 9161d2e529f..fcf535eda2b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -76,7 +76,7 @@ import org.mockito.Matchers; import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.OM_HARDLINK_FILE; -import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.fixFileName; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.fixFileName; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index d0a7d3ced32..d61a9f1c4fd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -42,7 +42,6 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -58,6 +57,8 @@ import java.util.stream.Stream; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.createHardLinkList; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.fixFileName; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; /** @@ -251,28 +252,4 @@ private void writeFilesToArchive(HashMap copyFiles, } } - - private Path createHardLinkList(int truncateLength, - Map hardLinkFiles) - throws IOException { - Path data = Files.createTempFile("data", "txt"); - StringBuilder sb = new StringBuilder(); - for (Map.Entry entry : hardLinkFiles.entrySet()) { - String fixedFile = fixFileName(truncateLength, entry.getValue()); - if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { - fixedFile = Paths.get(fixedFile).getFileName().toString(); - } - sb.append(fixFileName(truncateLength, entry.getKey())) - .append("\t") - .append(fixedFile) - .append("\n"); - } - Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); - return data; - } - - static String fixFileName(int truncateLength, Path file) { - return file.toString().substring(truncateLength); - } - } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 2cfdd2cadbb..09d23a9e8d9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; +import java.io.File; import java.io.IOException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -40,9 +41,17 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; @@ -270,4 +279,53 @@ private void verifySnapshotInfoForSnapDiff(final SnapshotInfo fromSnapshot, " should be older than to toSnapshot:" + toSnapshot.getName()); } } + + static void createHardLinks(Path dbPath) throws IOException { + File hardLinkFile = new File(dbPath.toString(), + OMDBCheckpointServlet.OM_HARDLINK_FILE); + if (hardLinkFile.exists()) { + List lines = + Files.lines(hardLinkFile.toPath()).collect(Collectors.toList()); + for (String l : lines) { + String from = l.split("\t")[1]; + String to = l.split("\t")[0]; + Path fixedFrom = fixName(dbPath, from); + Path fixedTo = fixName(dbPath, to); + Files.createLink(fixedTo, fixedFrom); + } + hardLinkFile.delete(); + } + } + + private static Path fixName(Path dbPath, String fileName) { + File file = new File(fileName); + if (!file.getName().equals(fileName)) { + return Paths.get(dbPath.getParent().toString(), fileName); + } + return Paths.get(dbPath.toString(), fileName); + } + + static Path createHardLinkList(int truncateLength, + Map hardLinkFiles) + throws IOException { + Path data = Files.createTempFile("data", "txt"); + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : hardLinkFiles.entrySet()) { + String fixedFile = fixFileName(truncateLength, entry.getValue()); + if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { + fixedFile = Paths.get(fixedFile).getFileName().toString(); + } + sb.append(fixFileName(truncateLength, entry.getKey())) + .append("\t") + .append(fixedFile) + .append("\n"); + } + Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); + return data; + } + + static String fixFileName(int truncateLength, Path file) { + return file.toString().substring(truncateLength); + } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index f80e48a70db..ed2ef434681 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3658,36 +3658,11 @@ private void moveOmSnapshotData(Path dbPath, Path dbSnapshotsDir) Path incomingSnapshotsDir = Paths.get(dbPath.toString(), OM_SNAPSHOT_DIR); if (incomingSnapshotsDir.toFile().exists()) { Files.move(incomingSnapshotsDir, dbSnapshotsDir); - createHardLinks(dbPath); + OmSnapshotManager.createHardLinks(dbPath); } } - private void createHardLinks(Path dbPath) throws IOException { - File hardLinkFile = new File(dbPath.toString(), - OMDBCheckpointServlet.OM_HARDLINK_FILE); - if (hardLinkFile.exists()) { - List lines = - Files.lines(hardLinkFile.toPath()).collect(Collectors.toList()); - for (String l : lines) { - String from = l.split("\t")[1]; - String to = l.split("\t")[0]; - Path fixedFrom = fixName(dbPath, from); - Path fixedTo = fixName(dbPath, to); - Files.createLink(fixedTo, fixedFrom); - } - hardLinkFile.delete(); - } - } - - private Path fixName(Path dbPath, String fileName) { - File file = new File(fileName); - if (!file.getName().equals(fileName)) { - return Paths.get(dbPath.getParent().toString(), fileName); - } - return Paths.get(dbPath.toString(), fileName); - } - /** * Re-instantiate MetadataManager with new DB checkpoint. * All the classes which use/ store MetadataManager should also be updated From 1ff9d66fca5751f842378dec156cabe91ec57029 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 4 Nov 2022 15:35:07 -0700 Subject: [PATCH 028/120] hard link test working --- .../ozone/om/TestOmSnapshotManager.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 3a89ed6973c..a21aa063df3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -29,13 +29,20 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.ozone.test.GenericTestUtils; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; +import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.OM_HARDLINK_FILE; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; @@ -114,6 +121,34 @@ public void testCloseOnEviction() throws IOException { verify(firstSnapshotStore, timeout(3000).times(1)).close(); } + @Test + public void testHardLinkCreation() throws IOException { + byte[] dummyData = {0}; + File dbDir = new File(testDir.toString(), "om.db"); + File snapDir1 = new File(testDir.toString(), "db.snapshots/dir1"); + File snapDir2 = new File(testDir.toString(), "db.snapshots/dir2"); + dbDir.mkdirs(); + snapDir1.mkdirs(); + snapDir2.mkdirs(); + Files.write(Paths.get(dbDir.toString(), "f1"), dummyData); + Files.write(Paths.get(snapDir1.toString(), "s1"), dummyData); + Map hardLinkFiles = new HashMap<>(); + hardLinkFiles.put(Paths.get(snapDir2.toString(), "f1"), + Paths.get(testDir.toString(), "db.checkpoints/dir1/f1")); + hardLinkFiles.put(Paths.get(snapDir2.toString(), "s1"), + Paths.get(snapDir1.toString(), "s1")); + + Path hardLinkList = + OmSnapshotManager.createHardLinkList(testDir.toString().length() + 1, hardLinkFiles); + Files.move(hardLinkList, Paths.get(dbDir.toString(), OM_HARDLINK_FILE)); + + OmSnapshotManager.createHardLinks(dbDir.toPath()); + + for (Path path : hardLinkFiles.keySet()) { + Assert.assertTrue(path.toFile().exists()); + } + } + private SnapshotInfo createSnapshotInfo() { String snapshotName = UUID.randomUUID().toString(); String volumeName = UUID.randomUUID().toString(); From 357c1e2fd6e43d37c2e67e36244a3450e7995205 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 8 Nov 2022 12:53:39 -0800 Subject: [PATCH 029/120] cleanup --- .../ozone/om/TestOMDbCheckpointServlet.java | 8 +++---- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 5 ++++ .../ozone/om/OMDBCheckpointServlet.java | 4 ++-- .../hadoop/ozone/om/OmSnapshotManager.java | 24 ++++++++++--------- .../apache/hadoop/ozone/om/OzoneManager.java | 1 + .../ozone/om/TestOmSnapshotManager.java | 20 ++++++++++++---- 6 files changed, 40 insertions(+), 22 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index fcf535eda2b..8c4e3baa920 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -76,7 +76,7 @@ import org.mockito.Matchers; import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.OM_HARDLINK_FILE; -import static org.apache.hadoop.ozone.om.OmSnapshotManager.fixFileName; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.truncateFileName; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; @@ -314,9 +314,9 @@ public void testWriteDbDataToStream() Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); String shortSnapshotLocation = - fixFileName(metaDirLength, Paths.get(snapshotDirName)); + truncateFileName(metaDirLength, Paths.get(snapshotDirName)); String shortSnapshotLocation2 = - fixFileName(metaDirLength, Paths.get(snapshotDirName2)); + truncateFileName(metaDirLength, Paths.get(snapshotDirName2)); Path finalSnapshotLocation = Paths.get(testDirName, shortSnapshotLocation); @@ -361,7 +361,7 @@ private Set getFiles(Path path, int truncateLength) for (Path file : files.collect(Collectors.toList())) { if (file != null) { if (!file.getFileName().toString().equals("dummyFile")) { - fileSet.add(fixFileName(truncateLength, file)); + fileSet.add(truncateFileName(truncateLength, file)); } } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 29c799d9e5e..e6173e67ab0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -599,6 +599,11 @@ public void testOmSnapshots() throws Exception { assertTrue(followerOM.getOmRatisServer().getLastAppliedTermIndex() .getTerm() >= leaderOMSnapshotTermIndex); + // Verify RPC server is running + GenericTestUtils.waitFor(() -> { + return followerOM.isOmRpcServerRunning(); + }, 100, 5000); + // Verify checkpoint installation was happened. String msg = "Reloaded OM state"; Assert.assertTrue(logCapture.getOutput().contains(msg)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index d61a9f1c4fd..8319e54325d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -58,7 +58,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.om.OmSnapshotManager.createHardLinkList; -import static org.apache.hadoop.ozone.om.OmSnapshotManager.fixFileName; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.truncateFileName; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; /** @@ -237,7 +237,7 @@ private void writeFilesToArchive(HashMap copyFiles, int truncateLength = metaDirPath.toString().length() + 1; for (Path file : copyFiles.values()) { - String fixedFile = fixFileName(truncateLength, file); + String fixedFile = truncateFileName(truncateLength, file); if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { // checkpoint files go to root of tarball fixedFile = Paths.get(fixedFile).getFileName().toString(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 09d23a9e8d9..e8dedd4abe4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -289,20 +289,22 @@ static void createHardLinks(Path dbPath) throws IOException { for (String l : lines) { String from = l.split("\t")[1]; String to = l.split("\t")[0]; - Path fixedFrom = fixName(dbPath, from); - Path fixedTo = fixName(dbPath, to); - Files.createLink(fixedTo, fixedFrom); + Path fullFromPath = getFullPath(dbPath, from); + Path fullToPath = getFullPath(dbPath, to); + Files.createLink(fullToPath, fullFromPath); } hardLinkFile.delete(); } } - private static Path fixName(Path dbPath, String fileName) { + private static Path getFullPath(Path dbPath, String fileName) { File file = new File(fileName); - if (!file.getName().equals(fileName)) { - return Paths.get(dbPath.getParent().toString(), fileName); + // If there is no directory then this file belongs in the db + if (file.getName().equals(fileName)) { + return Paths.get(dbPath.toString(), fileName); } - return Paths.get(dbPath.toString(), fileName); + // else this file belong in a directory parallel to the db + return Paths.get(dbPath.getParent().toString(), fileName); } static Path createHardLinkList(int truncateLength, @@ -311,11 +313,12 @@ static Path createHardLinkList(int truncateLength, Path data = Files.createTempFile("data", "txt"); StringBuilder sb = new StringBuilder(); for (Map.Entry entry : hardLinkFiles.entrySet()) { - String fixedFile = fixFileName(truncateLength, entry.getValue()); + String fixedFile = truncateFileName(truncateLength, entry.getValue()); + // If this file is from the active db, strip the path if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { fixedFile = Paths.get(fixedFile).getFileName().toString(); } - sb.append(fixFileName(truncateLength, entry.getKey())) + sb.append(truncateFileName(truncateLength, entry.getKey())) .append("\t") .append(fixedFile) .append("\n"); @@ -324,8 +327,7 @@ static Path createHardLinkList(int truncateLength, return data; } - static String fixFileName(int truncateLength, Path file) { + static String truncateFileName(int truncateLength, Path file) { return file.toString().substring(truncateLength); } - } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index ed2ef434681..acc682fd761 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3653,6 +3653,7 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, return dbBackupDir; } + // move the new snapshot directory into place and create hard links private void moveOmSnapshotData(Path dbPath, Path dbSnapshotsDir) throws IOException { Path incomingSnapshotsDir = Paths.get(dbPath.toString(), OM_SNAPSHOT_DIR); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index a21aa063df3..be794df8f3a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -42,6 +42,10 @@ import java.util.Map; import java.util.UUID; +import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.OM_HARDLINK_FILE; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; import static org.mockito.Mockito.mock; @@ -124,9 +128,14 @@ public void testCloseOnEviction() throws IOException { @Test public void testHardLinkCreation() throws IOException { byte[] dummyData = {0}; - File dbDir = new File(testDir.toString(), "om.db"); - File snapDir1 = new File(testDir.toString(), "db.snapshots/dir1"); - File snapDir2 = new File(testDir.toString(), "db.snapshots/dir2"); + File dbDir = new File(testDir.toString(), OM_DB_NAME); + File snapDir1 = new File(testDir.toString(), + OM_SNAPSHOT_DIR + OM_KEY_PREFIX + "dir1"); + File snapDir2 = new File(testDir.toString(), + OM_SNAPSHOT_DIR + OM_KEY_PREFIX + "dir2"); + File checkpointDir1 = new File(testDir.toString(), + OM_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); + dbDir.mkdirs(); snapDir1.mkdirs(); snapDir2.mkdirs(); @@ -134,12 +143,13 @@ public void testHardLinkCreation() throws IOException { Files.write(Paths.get(snapDir1.toString(), "s1"), dummyData); Map hardLinkFiles = new HashMap<>(); hardLinkFiles.put(Paths.get(snapDir2.toString(), "f1"), - Paths.get(testDir.toString(), "db.checkpoints/dir1/f1")); + Paths.get(checkpointDir1.toString(), "f1")); hardLinkFiles.put(Paths.get(snapDir2.toString(), "s1"), Paths.get(snapDir1.toString(), "s1")); Path hardLinkList = - OmSnapshotManager.createHardLinkList(testDir.toString().length() + 1, hardLinkFiles); + OmSnapshotManager.createHardLinkList( + testDir.toString().length() + 1, hardLinkFiles); Files.move(hardLinkList, Paths.get(dbDir.toString(), OM_HARDLINK_FILE)); OmSnapshotManager.createHardLinks(dbDir.toPath()); From 4948396d797ecab1014cdc4c557ee3100e3c12a4 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 8 Nov 2022 13:00:10 -0800 Subject: [PATCH 030/120] sleep for double buffer --- .../java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index e6173e67ab0..965b0841ee6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -559,6 +559,8 @@ public void testOmSnapshots() throws Exception { // Do some transactions so that the log index increases List keys = writeKeysToIncreaseLogIndex(leaderRatisServer, 200); + // Avoid double buffer issue + Thread.sleep(5000); objectStore.createSnapshot(volumeName, bucketName, "snap1"); // Get the latest db checkpoint from the leader OM. From 0caf85b7f20782f63408bb4444b250ef6ac54890 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 8 Nov 2022 16:01:57 -0800 Subject: [PATCH 031/120] cleanup --- .../apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 10 +++++++++- .../apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 3 +++ .../org/apache/hadoop/ozone/om/OmSnapshotManager.java | 9 +++++++++ .../om/snapshot/OzoneManagerSnapshotProvider.java | 2 +- .../apache/hadoop/ozone/om/TestOmSnapshotManager.java | 7 +++++++ 5 files changed, 29 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 965b0841ee6..3ee81805581 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -51,6 +51,7 @@ import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithData.createKey; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import org.assertj.core.api.Fail; import org.junit.Assert; @@ -563,6 +564,8 @@ public void testOmSnapshots() throws Exception { Thread.sleep(5000); objectStore.createSnapshot(volumeName, bucketName, "snap1"); + // allow the snapshot to be written to the info table + Thread.sleep(5000); // Get the latest db checkpoint from the leader OM. TransactionInfo transactionInfo = TransactionInfo.readTransactionInfo(leaderOM.getMetadataManager()); @@ -615,7 +618,12 @@ public void testOmSnapshots() throws Exception { .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(".snapshot/snap1/" + keys.get(0)).build(); - OmKeyInfo omKeyInfo = followerOM.lookupKey(omKeyArgs); + OmKeyInfo omKeyInfo = null; + try { + omKeyInfo = followerOM.lookupKey(omKeyArgs); + } catch (Exception e) { + fail("received exception: " + e); + } Assert.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 8319e54325d..3420838692a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -157,6 +157,9 @@ private List getSnapshotDirs(DBCheckpoint checkpoint) list.add(path); } } + if (list.size() == 0) { + LOG.error("gbj failure"); + } return list; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index e8dedd4abe4..c6f022eff39 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -280,6 +280,7 @@ private void verifySnapshotInfoForSnapDiff(final SnapshotInfo fromSnapshot, } } + // Create hard links listed in OM_HARDLINK_FILE static void createHardLinks(Path dbPath) throws IOException { File hardLinkFile = new File(dbPath.toString(), OMDBCheckpointServlet.OM_HARDLINK_FILE); @@ -297,6 +298,7 @@ static void createHardLinks(Path dbPath) throws IOException { } } + // Get the full path of OM_HARDLINK_FILE entry private static Path getFullPath(Path dbPath, String fileName) { File file = new File(fileName); // If there is no directory then this file belongs in the db @@ -307,6 +309,13 @@ private static Path getFullPath(Path dbPath, String fileName) { return Paths.get(dbPath.getParent().toString(), fileName); } + // Create list of links to add to follower + // Format of entries are either: + // db.snapshot/dir1/fileTo fileFrom + // for files in active db or: + // db.snapshot/dir1/fileTo parent/dir2/fileFrom + // for files in another directory, (either another snapshot dir or + // sst compaction backup directory) static Path createHardLinkList(int truncateLength, Map hardLinkFiles) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java index 3ef2661f822..3d428b65a02 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java @@ -149,7 +149,7 @@ public DBCheckpoint getOzoneManagerDBSnapshot(String leaderOMNodeID) // Untar the checkpoint file. Path untarredDbDir = Paths.get(snapshotFilePath); FileUtil.unTar(targetFile, untarredDbDir.toFile()); - FileUtils.deleteQuietly(targetFile); + // FileUtils.deleteQuietly(targetFile); LOG.info("Successfully downloaded latest checkpoint from leader OM: {}", leaderOMNodeID); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index be794df8f3a..7d06ad0c865 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -128,6 +128,7 @@ public void testCloseOnEviction() throws IOException { @Test public void testHardLinkCreation() throws IOException { byte[] dummyData = {0}; + File dbDir = new File(testDir.toString(), OM_DB_NAME); File snapDir1 = new File(testDir.toString(), OM_SNAPSHOT_DIR + OM_KEY_PREFIX + "dir1"); @@ -136,24 +137,30 @@ public void testHardLinkCreation() throws IOException { File checkpointDir1 = new File(testDir.toString(), OM_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); + // Create dummy files to be linked to dbDir.mkdirs(); snapDir1.mkdirs(); snapDir2.mkdirs(); Files.write(Paths.get(dbDir.toString(), "f1"), dummyData); Files.write(Paths.get(snapDir1.toString(), "s1"), dummyData); + + // Create map of links to dummy files Map hardLinkFiles = new HashMap<>(); hardLinkFiles.put(Paths.get(snapDir2.toString(), "f1"), Paths.get(checkpointDir1.toString(), "f1")); hardLinkFiles.put(Paths.get(snapDir2.toString(), "s1"), Paths.get(snapDir1.toString(), "s1")); + // Create link list Path hardLinkList = OmSnapshotManager.createHardLinkList( testDir.toString().length() + 1, hardLinkFiles); Files.move(hardLinkList, Paths.get(dbDir.toString(), OM_HARDLINK_FILE)); + // Create links from list OmSnapshotManager.createHardLinks(dbDir.toPath()); + // Confirm expected links for (Path path : hardLinkFiles.keySet()) { Assert.assertTrue(path.toFile().exists()); } From 91f0a90eff7b52588424c2c4174e58b4f442f334 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 8 Nov 2022 16:39:22 -0800 Subject: [PATCH 032/120] cleanup --- .../ozone/om/TestOMDbCheckpointServlet.java | 2 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 26 ++++++++++++++++--- .../ozone/om/OMDBCheckpointServlet.java | 7 +---- .../hadoop/ozone/om/OmSnapshotManager.java | 3 ++- .../ozone/om/TestOmSnapshotManager.java | 2 +- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 8c4e3baa920..36d6beb8684 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -75,7 +75,7 @@ import org.junit.rules.Timeout; import org.mockito.Matchers; -import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.OM_HARDLINK_FILE; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE; import static org.apache.hadoop.ozone.om.OmSnapshotManager.truncateFileName; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.mockito.ArgumentMatchers.any; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 3ee81805581..bae59417793 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.hdds.ExitManager; @@ -66,7 +67,7 @@ /** * Tests the Ratis snapshots feature in OM. */ -@Timeout(5000) +@Timeout(500) public class TestOMRatisSnapshots { private MiniOzoneHAClusterImpl cluster = null; @@ -560,12 +561,29 @@ public void testOmSnapshots() throws Exception { // Do some transactions so that the log index increases List keys = writeKeysToIncreaseLogIndex(leaderRatisServer, 200); - // Avoid double buffer issue - Thread.sleep(5000); + // Avoid double buffer issue waiting for keys + GenericTestUtils.waitFor(() -> { + try { + leaderOM.getMetadataManager().getKeyTable(BucketLayout.OBJECT_STORE) + .getSkipCache(keys.get(keys.size()-1)); + return true; + } catch (Exception e) { + return false; + } + }, 100, 3000); objectStore.createSnapshot(volumeName, bucketName, "snap1"); // allow the snapshot to be written to the info table - Thread.sleep(5000); + GenericTestUtils.waitFor(() -> { + try { + leaderOM.getMetadataManager().getSnapshotInfoTable() + .getSkipCache( + SnapshotInfo.getTableKey(volumeName, bucketName, "snap1")); + return true; + } catch (Exception e) { + return false; + } + }, 100, 3000); // Get the latest db checkpoint from the leader OM. TransactionInfo transactionInfo = TransactionInfo.readTransactionInfo(leaderOM.getMetadataManager()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 3420838692a..c5c6aa6febd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -81,8 +81,6 @@ public class OMDBCheckpointServlet extends DBCheckpointServlet { private static final long serialVersionUID = 1L; private static final String DURATION_TO_WAIT_FOR_DIRECTORY = "PT10S"; - public static final String OM_HARDLINK_FILE = "hardLinkFile"; - @Override public void init() throws ServletException { @@ -157,9 +155,6 @@ private List getSnapshotDirs(DBCheckpoint checkpoint) list.add(path); } } - if (list.size() == 0) { - LOG.error("gbj failure"); - } return list; } @@ -250,7 +245,7 @@ private void writeFilesToArchive(HashMap copyFiles, } if (!hardLinkFiles.isEmpty()) { Path hardLinkFile = createHardLinkList(truncateLength, hardLinkFiles); - includeFile(hardLinkFile.toFile(), OM_HARDLINK_FILE, + includeFile(hardLinkFile.toFile(), OmSnapshotManager.OM_HARDLINK_FILE, archiveOutputStream); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index c6f022eff39..60e52382894 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -64,6 +64,7 @@ * This class is used to manage/create OM snapshots. */ public final class OmSnapshotManager { + public static final String OM_HARDLINK_FILE = "hardLinkFile"; private final OzoneManager ozoneManager; private final SnapshotDiffManager snapshotDiffManager; private final LoadingCache snapshotCache; @@ -283,7 +284,7 @@ private void verifySnapshotInfoForSnapDiff(final SnapshotInfo fromSnapshot, // Create hard links listed in OM_HARDLINK_FILE static void createHardLinks(Path dbPath) throws IOException { File hardLinkFile = new File(dbPath.toString(), - OMDBCheckpointServlet.OM_HARDLINK_FILE); + OM_HARDLINK_FILE); if (hardLinkFile.exists()) { List lines = Files.lines(hardLinkFile.toPath()).collect(Collectors.toList()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 7d06ad0c865..2c91ba38f43 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -46,7 +46,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; -import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.OM_HARDLINK_FILE; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; From 8d8d521103a4a18fe63bca18803c4e2631d648fb Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 8 Nov 2022 17:16:04 -0800 Subject: [PATCH 033/120] cleanup --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 31 ++++++++++--------- .../apache/hadoop/ozone/om/OzoneManager.java | 2 +- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index bae59417793..cb1b0e588af 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -50,6 +50,7 @@ import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.server.protocol.TermIndex; +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithData.createKey; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -58,6 +59,7 @@ import org.junit.Assert; import org.junit.Ignore; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -564,22 +566,24 @@ public void testOmSnapshots() throws Exception { // Avoid double buffer issue waiting for keys GenericTestUtils.waitFor(() -> { try { - leaderOM.getMetadataManager().getKeyTable(BucketLayout.OBJECT_STORE) - .getSkipCache(keys.get(keys.size()-1)); - return true; + OmKeyInfo key = leaderOM.getMetadataManager() + .getKeyTable(ozoneBucket.getBucketLayout()) + .getSkipCache(leaderOM.getMetadataManager().getOzoneKey(volumeName, bucketName, keys.get(0))); + return key != null; } catch (Exception e) { return false; } - }, 100, 3000); + }, 100, 10000); objectStore.createSnapshot(volumeName, bucketName, "snap1"); // allow the snapshot to be written to the info table GenericTestUtils.waitFor(() -> { try { - leaderOM.getMetadataManager().getSnapshotInfoTable() - .getSkipCache( - SnapshotInfo.getTableKey(volumeName, bucketName, "snap1")); - return true; + SnapshotInfo snapshotInfo = + leaderOM.getMetadataManager().getSnapshotInfoTable() + .getSkipCache( + SnapshotInfo.getTableKey(volumeName, bucketName, "snap1")); + return snapshotInfo != null; } catch (Exception e) { return false; } @@ -603,10 +607,9 @@ public void testOmSnapshots() throws Exception { // The recently started OM should be lagging behind the leader OM. // Wait & for follower to update transactions to leader snapshot index. // Timeout error if follower does not load update within 3s - GenericTestUtils.waitFor(() -> { - return followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex() - >= leaderOMSnapshotIndex - 1; - }, 100, 3000); + GenericTestUtils.waitFor(() -> + followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex() + >= leaderOMSnapshotIndex - 1, 100, 3000); long followerOMLastAppliedIndex = followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex(); @@ -640,9 +643,9 @@ public void testOmSnapshots() throws Exception { try { omKeyInfo = followerOM.lookupKey(omKeyArgs); } catch (Exception e) { - fail("received exception: " + e); + Assertions.fail("received exception: " + e); } - Assert.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); + Assertions.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); } private List writeKeysToIncreaseLogIndex( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index acc682fd761..9af672fbda9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3604,7 +3604,7 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, File dbBackupDir = new File(dbDir, dbBackupName); if (!dbBackupDir.mkdirs()) { throw new IOException("Failed to make db backup dir: " + - dbBackupDir.toString()); + dbBackupDir); } File dbBackup = new File(dbBackupDir, oldDB.getName()); File dbSnapshotsDir = new File(dbDir, OM_SNAPSHOT_DIR); From ba6b64970860a45b1fa70ae49a7403ae4f57de25 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 8 Nov 2022 17:31:25 -0800 Subject: [PATCH 034/120] checkstyle --- .../ozone/om/TestOMDbCheckpointServlet.java | 4 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 68 +++++++++---------- .../ozone/om/OMDBCheckpointServlet.java | 2 +- .../apache/hadoop/ozone/om/OzoneManager.java | 2 - .../OzoneManagerSnapshotProvider.java | 2 +- 5 files changed, 37 insertions(+), 41 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 36d6beb8684..ebcc69ac1d2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -286,8 +286,8 @@ public void testWriteDbDataToStream() prepArchiveData(); try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { - omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, - fileOutputStream); + omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, + fileOutputStream); } // Untar the file into a temp folder to be examined diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index cb1b0e588af..8e7be53e6f7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -16,19 +16,11 @@ */ package org.apache.hadoop.ozone.om; -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdds.ExitManager; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; @@ -38,7 +30,6 @@ import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.VolumeArgs; -import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -46,15 +37,8 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; -import org.apache.hadoop.hdds.ExitManager; import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.server.protocol.TermIndex; - -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithData.createKey; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import org.assertj.core.api.Fail; import org.junit.Assert; import org.junit.Ignore; @@ -66,6 +50,19 @@ import org.slf4j.Logger; import org.slf4j.event.Level; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithData.createKey; +import static org.junit.Assert.assertTrue; + /** * Tests the Ratis snapshots feature in OM. */ @@ -565,28 +562,29 @@ public void testOmSnapshots() throws Exception { // Avoid double buffer issue waiting for keys GenericTestUtils.waitFor(() -> { - try { - OmKeyInfo key = leaderOM.getMetadataManager() - .getKeyTable(ozoneBucket.getBucketLayout()) - .getSkipCache(leaderOM.getMetadataManager().getOzoneKey(volumeName, bucketName, keys.get(0))); - return key != null; - } catch (Exception e) { - return false; - } + try { + OmKeyInfo key = leaderOM.getMetadataManager() + .getKeyTable(ozoneBucket.getBucketLayout()) + .getSkipCache(leaderOM.getMetadataManager() + .getOzoneKey(volumeName, bucketName, keys.get(0))); + return key != null; + } catch (Exception e) { + return false; + } }, 100, 10000); objectStore.createSnapshot(volumeName, bucketName, "snap1"); // allow the snapshot to be written to the info table GenericTestUtils.waitFor(() -> { - try { - SnapshotInfo snapshotInfo = - leaderOM.getMetadataManager().getSnapshotInfoTable() - .getSkipCache( - SnapshotInfo.getTableKey(volumeName, bucketName, "snap1")); - return snapshotInfo != null; - } catch (Exception e) { - return false; - } + try { + SnapshotInfo snapshotInfo = + leaderOM.getMetadataManager().getSnapshotInfoTable() + .getSkipCache( + SnapshotInfo.getTableKey(volumeName, bucketName, "snap1")); + return snapshotInfo != null; + } catch (Exception e) { + return false; + } }, 100, 3000); // Get the latest db checkpoint from the leader OM. TransactionInfo transactionInfo = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index c5c6aa6febd..c8d48c77926 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -195,7 +195,7 @@ private void processDir(Path dir, Map copyFiles, @Override public void writeDbDataToStream(DBCheckpoint checkpoint, - OutputStream destination) + OutputStream destination) throws IOException, InterruptedException, CompressorException { // Map of inodes to path HashMap copyFiles = new HashMap<>(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 9af672fbda9..d9987b7e42c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -49,7 +49,6 @@ import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import com.google.common.base.Optional; import org.apache.hadoop.conf.Configuration; @@ -233,7 +232,6 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE; import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; import static org.apache.hadoop.ozone.OzoneConsts.PREPARE_MARKER_KEY; import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java index 3d428b65a02..3ef2661f822 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java @@ -149,7 +149,7 @@ public DBCheckpoint getOzoneManagerDBSnapshot(String leaderOMNodeID) // Untar the checkpoint file. Path untarredDbDir = Paths.get(snapshotFilePath); FileUtil.unTar(targetFile, untarredDbDir.toFile()); - // FileUtils.deleteQuietly(targetFile); + FileUtils.deleteQuietly(targetFile); LOG.info("Successfully downloaded latest checkpoint from leader OM: {}", leaderOMNodeID); From c3eb22b56706a58eea5c421051359771632e8b9a Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 9 Nov 2022 08:14:37 -0800 Subject: [PATCH 035/120] findbugs --- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 3 +++ .../apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 4 ++++ .../hadoop/ozone/om/OMDBCheckpointServlet.java | 2 ++ .../apache/hadoop/ozone/om/OmSnapshotManager.java | 8 +++++++- .../hadoop/ozone/om/TestOmSnapshotManager.java | 12 +++++++++--- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index ebcc69ac1d2..40f500e86cb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -41,6 +41,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; @@ -354,6 +355,7 @@ public void testWriteDbDataToStream() initialSnapshotSet, finalSnapshotSet); } + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private Set getFiles(Path path, int truncateLength) throws IOException { Set fileSet = new HashSet<>(); @@ -371,6 +373,7 @@ private Set getFiles(Path path, int truncateLength) // tests to see that dummy entry in hardlink file looks something like: // "dir1/dummyFile dir2/dummyFile" + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private void checkDummyFile(String dir0, String dir1, String line, String testDirName) { String[] files = line.split("\t"); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 8e7be53e6f7..5df73198640 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.om; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.ExitManager; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -541,6 +542,8 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { } @Test + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE", + "NP_NULL_ON_SOME_PATH"}) public void testOmSnapshots() throws Exception { // Get the leader OM String leaderOMNodeId = OmFailoverProxyUtil @@ -643,6 +646,7 @@ public void testOmSnapshots() throws Exception { } catch (Exception e) { Assertions.fail("received exception: " + e); } + Assertions.assertTrue(omKeyInfo != null); Assertions.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index c8d48c77926..176adbf3132 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -20,6 +20,7 @@ import javax.servlet.ServletException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.compressors.CompressorException; @@ -222,6 +223,7 @@ public void writeDbDataToStream(DBCheckpoint checkpoint, } } + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private void writeFilesToArchive(HashMap copyFiles, HashMap hardLinkFiles, ArchiveOutputStream archiveOutputStream) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 60e52382894..5484aff39fb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -22,6 +22,7 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.cache.RemovalListener; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -295,11 +296,15 @@ static void createHardLinks(Path dbPath) throws IOException { Path fullToPath = getFullPath(dbPath, to); Files.createLink(fullToPath, fullFromPath); } - hardLinkFile.delete(); + if (!hardLinkFile.delete()) { + throw new IOException( + "Failed to delete: " + hardLinkFile.toString()); + } } } // Get the full path of OM_HARDLINK_FILE entry + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private static Path getFullPath(Path dbPath, String fileName) { File file = new File(fileName); // If there is no directory then this file belongs in the db @@ -317,6 +322,7 @@ private static Path getFullPath(Path dbPath, String fileName) { // db.snapshot/dir1/fileTo parent/dir2/fileFrom // for files in another directory, (either another snapshot dir or // sst compaction backup directory) + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) static Path createHardLinkList(int truncateLength, Map hardLinkFiles) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 2c91ba38f43..0e90f98b5c1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -138,9 +138,15 @@ public void testHardLinkCreation() throws IOException { OM_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); // Create dummy files to be linked to - dbDir.mkdirs(); - snapDir1.mkdirs(); - snapDir2.mkdirs(); + if (!dbDir.mkdirs()) { + throw new IOException("failed to make directory: " + dbDir.toString()); + } + if (!snapDir1.mkdirs()) { + throw new IOException("failed to make directory: " + snapDir1.toString()); + } + if (!snapDir2.mkdirs()) { + throw new IOException("failed to make directory: " + dbDir.toString()); + } Files.write(Paths.get(dbDir.toString(), "f1"), dummyData); Files.write(Paths.get(snapDir1.toString(), "s1"), dummyData); From c3e6e413916f64731b7ac7c39d36a9b7a3111383 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 9 Nov 2022 17:28:41 -0800 Subject: [PATCH 036/120] includeSnapshotData initally working --- .../org/apache/hadoop/ozone/OzoneConsts.java | 2 ++ .../hdds/utils/DBCheckpointServlet.java | 3 ++- .../ozone/om/helpers/OMNodeDetails.java | 7 ++++-- .../hdds/scm/TestSCMDbCheckpointServlet.java | 2 +- .../ozone/om/TestOMDbCheckpointServlet.java | 11 +++++--- .../ozone/om/OMDBCheckpointServlet.java | 25 ++++++++++++++++--- 6 files changed, 40 insertions(+), 10 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 67544babc12..e49a0a4a56a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -140,6 +140,8 @@ public final class OzoneConsts { public static final String STORAGE_DIR_CHUNKS = "chunks"; public static final String OZONE_DB_CHECKPOINT_REQUEST_FLUSH = "flushBeforeCheckpoint"; + public static final String OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA = + "includeSnapshotData"; public static final String RANGER_OZONE_SERVICE_VERSION_KEY = "#RANGEROZONESERVICEVERSION"; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index d46fcfc2bb5..6568a44ae75 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -174,7 +174,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { file.toString() + ".tgz\""); Instant start = Instant.now(); - writeDbDataToStream(checkpoint, + writeDbDataToStream(checkpoint, request, response.getOutputStream()); Instant end = Instant.now(); @@ -201,6 +201,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { } public void writeDbDataToStream(DBCheckpoint checkpoint, + HttpServletRequest ignoredRequest, OutputStream destination) throws IOException, InterruptedException, CompressorException { writeDBCheckpointToStream(checkpoint, destination); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java index 1a06601588f..ee680ef854e 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.net.InetSocketAddress; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; @@ -164,13 +165,15 @@ public String getOMDBCheckpointEnpointUrl(boolean isHttpPolicy) { if (StringUtils.isNotEmpty(getHttpAddress())) { return "http://" + getHttpAddress() + OZONE_DB_CHECKPOINT_HTTP_ENDPOINT + - "?" + OZONE_DB_CHECKPOINT_REQUEST_FLUSH + "=true"; + "?" + OZONE_DB_CHECKPOINT_REQUEST_FLUSH + "=true&" + + OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA + "=true"; } } else { if (StringUtils.isNotEmpty(getHttpsAddress())) { return "https://" + getHttpsAddress() + OZONE_DB_CHECKPOINT_HTTP_ENDPOINT + - "?" + OZONE_DB_CHECKPOINT_REQUEST_FLUSH + "=true"; + "?" + OZONE_DB_CHECKPOINT_REQUEST_FLUSH + "=true&" + + OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA + "=true"; } } return null; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java index 3398e0d22aa..9ffda5a5aab 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java @@ -120,7 +120,7 @@ public void testDoGet() Collections.emptyList(), false); doCallRealMethod().when(scmDbCheckpointServletMock) - .writeDbDataToStream(any(), any()); + .writeDbDataToStream(any(), any(), any()); HttpServletRequest requestMock = mock(HttpServletRequest.class); HttpServletResponse responseMock = mock(HttpServletResponse.class); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 40f500e86cb..a7334ebb7b9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -61,6 +61,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE; @@ -184,7 +185,7 @@ private void setupCluster() throws Exception { responseMock); doCallRealMethod().when(omDbCheckpointServletMock) - .writeDbDataToStream(any(), any()); + .writeDbDataToStream(any(), any(), any()); } @Test @@ -285,9 +286,8 @@ public void testSpnegoEnabled() throws Exception { public void testWriteDbDataToStream() throws Exception { prepArchiveData(); - try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { - omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, + omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock, fileOutputStream); } @@ -463,5 +463,10 @@ private void prepArchiveData() throws Exception { dbCheckpoint = cluster.getOzoneManager() .getMetadataManager().getStore() .getCheckpoint(true); + when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)) + .thenReturn("true"); } + + + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 176adbf3132..b58697430fd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om; import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.compress.archivers.ArchiveOutputStream; @@ -26,6 +27,7 @@ import org.apache.commons.compress.compressors.CompressorException; import org.apache.commons.compress.compressors.CompressorOutputStream; import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.recon.ReconConfig; import org.apache.hadoop.hdds.server.ServerUtils; @@ -58,6 +60,7 @@ import java.util.stream.Stream; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.om.OmSnapshotManager.createHardLinkList; import static org.apache.hadoop.ozone.om.OmSnapshotManager.truncateFileName; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; @@ -116,7 +119,8 @@ public void init() throws ServletException { private void getFilesForArchive(DBCheckpoint checkpoint, Map copyFiles, - Map hardLinkFiles) + Map hardLinkFiles, + boolean includeSnapshotData) throws IOException, InterruptedException { Path dir = checkpoint.getCheckpointLocation(); @@ -128,6 +132,10 @@ private void getFilesForArchive(DBCheckpoint checkpoint, copyFiles.put(key, file); } } + if (!includeSnapshotData) { + return; + } + for (Path snapshotDir : getSnapshotDirs(checkpoint)) { processDir(snapshotDir, copyFiles, hardLinkFiles); } @@ -196,14 +204,16 @@ private void processDir(Path dir, Map copyFiles, @Override public void writeDbDataToStream(DBCheckpoint checkpoint, - OutputStream destination) + HttpServletRequest request, + OutputStream destination) throws IOException, InterruptedException, CompressorException { // Map of inodes to path HashMap copyFiles = new HashMap<>(); // Map of link to path HashMap hardLinkFiles = new HashMap<>(); - getFilesForArchive(checkpoint, copyFiles, hardLinkFiles); + getFilesForArchive(checkpoint, copyFiles, hardLinkFiles, + includeSnapshotData(request)); try (CompressorOutputStream gzippedOut = new CompressorStreamFactory() .createCompressorOutputStream(CompressorStreamFactory.GZIP, @@ -223,6 +233,15 @@ public void writeDbDataToStream(DBCheckpoint checkpoint, } } + private boolean includeSnapshotData(HttpServletRequest request) { + String includeParam = + request.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA); + if (StringUtils.isNotEmpty(includeParam)) { + return Boolean.parseBoolean(includeParam); + } + return false; + } + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private void writeFilesToArchive(HashMap copyFiles, HashMap hardLinkFiles, From 71782b042c36274880fdaeac09e9a7115339c93b Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 11 Nov 2022 11:05:05 -0800 Subject: [PATCH 037/120] inc timeout --- .../java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 5df73198640..7bfd0d06cc3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -67,7 +67,7 @@ /** * Tests the Ratis snapshots feature in OM. */ -@Timeout(500) +@Timeout(5000) public class TestOMRatisSnapshots { private MiniOzoneHAClusterImpl cluster = null; From 8acbc074d2daa8ea03c86e08e4391f296f0c0dcb Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 11 Nov 2022 13:57:24 -0800 Subject: [PATCH 038/120] added test for includeSnapshotData is false --- .../ozone/om/TestOMDbCheckpointServlet.java | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index a7334ebb7b9..d4695ec6f30 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -286,6 +286,9 @@ public void testSpnegoEnabled() throws Exception { public void testWriteDbDataToStream() throws Exception { prepArchiveData(); + // Do include snapshot data + when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)) + .thenReturn("true"); try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock, fileOutputStream); @@ -355,6 +358,40 @@ public void testWriteDbDataToStream() initialSnapshotSet, finalSnapshotSet); } + @Test + public void testWriteDbDataWithoutOmSnapshot() + throws Exception { + prepArchiveData(); + + // Don't include snapshot data + when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)) + .thenReturn(null); + try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { + omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock, + fileOutputStream); + } + + // Untar the file into a temp folder to be examined + String testDirName = folder.newFolder().getAbsolutePath(); + int testDirLength = testDirName.length() + 1; + FileUtil.unTar(tempFile, new File(testDirName)); + + Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); + int metaDirLength = metaDir.toString().length() + 1; + Path finalCheckpointLocation = + Paths.get(testDirName); + + // Confirm the checkpoint directories match, (after remove extras) + Set initialCheckpointSet = getFiles(checkpointLocation, + checkpointLocation.toString().length() + 1); + Set finalCheckpointSet = getFiles(finalCheckpointLocation, + testDirLength); + + Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); + } + + + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private Set getFiles(Path path, int truncateLength) throws IOException { @@ -463,8 +500,6 @@ private void prepArchiveData() throws Exception { dbCheckpoint = cluster.getOzoneManager() .getMetadataManager().getStore() .getCheckpoint(true); - when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)) - .thenReturn("true"); } From aee7bbfc533886c0c75c8b3fb66b3ef23cbfacb4 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 11 Nov 2022 16:15:08 -0800 Subject: [PATCH 039/120] renamed OzoneManagerSnapshotProvider --- .../apache/hadoop/ozone/om/OzoneManager.java | 26 +++++++++---------- .../OmRatisSnapshotProvider.java} | 26 +++++++++++++------ 2 files changed, 31 insertions(+), 21 deletions(-) rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{snapshot/OzoneManagerSnapshotProvider.java => ratis_snapshot/OmRatisSnapshotProvider.java} (85%) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index d9987b7e42c..2ecab6c3af0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -76,6 +76,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.multitenant.OMRangerBGSyncService; +import org.apache.hadoop.ozone.om.ratis_snapshot.OmRatisSnapshotProvider; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.util.OzoneNetUtils; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -153,7 +154,6 @@ import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; -import org.apache.hadoop.ozone.om.snapshot.OzoneManagerSnapshotProvider; import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; import org.apache.hadoop.ozone.om.upgrade.OMUpgradeFinalizer; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OzoneManagerAdminService; @@ -362,7 +362,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private boolean isRatisEnabled; private OzoneManagerRatisServer omRatisServer; - private OzoneManagerSnapshotProvider omSnapshotProvider; + private OmRatisSnapshotProvider omRatisSnapshotProvider; private OMNodeDetails omNodeDetails; private Map peerNodesMap; private File omRatisSnapshotDir; @@ -1334,7 +1334,7 @@ private void initializeRatisDirs(OzoneConfiguration conf) throws IOException { } if (peerNodesMap != null && !peerNodesMap.isEmpty()) { - this.omSnapshotProvider = new OzoneManagerSnapshotProvider( + this.omRatisSnapshotProvider = new OmRatisSnapshotProvider( configuration, omRatisSnapshotDir, peerNodesMap); } } @@ -1370,8 +1370,8 @@ public OzoneManagerRatisServer getOmRatisServer() { } @VisibleForTesting - public OzoneManagerSnapshotProvider getOmSnapshotProvider() { - return omSnapshotProvider; + public OmRatisSnapshotProvider getOmSnapshotProvider() { + return omRatisSnapshotProvider; } @VisibleForTesting @@ -1808,11 +1808,11 @@ private void addOMNodeToPeers(String newOMNodeId) throws IOException { exitManager.exitSystem(1, e.getLocalizedMessage(), e, LOG); } - if (omSnapshotProvider == null) { - omSnapshotProvider = new OzoneManagerSnapshotProvider( + if (omRatisSnapshotProvider == null) { + omRatisSnapshotProvider = new OmRatisSnapshotProvider( configuration, omRatisSnapshotDir, peerNodesMap); } else { - omSnapshotProvider.addNewPeerNode(newOMNodeDetails); + omRatisSnapshotProvider.addNewPeerNode(newOMNodeDetails); } omRatisServer.addRaftPeer(newOMNodeDetails); peerNodesMap.put(newOMNodeId, newOMNodeDetails); @@ -1831,7 +1831,7 @@ private void removeOMNodeFromPeers(String decommNodeId) throws IOException { "present in peer list"); } - omSnapshotProvider.removeDecommissionedPeerNode(decommNodeId); + omRatisSnapshotProvider.removeDecommissionedPeerNode(decommNodeId); omRatisServer.removeRaftPeer(decommOMNodeDetails); peerNodesMap.remove(decommNodeId); LOG.info("Removed OM {} from OM Peer Nodes.", decommNodeId); @@ -2057,8 +2057,8 @@ public void stop() { if (jvmPauseMonitor != null) { jvmPauseMonitor.stop(); } - if (omSnapshotProvider != null) { - omSnapshotProvider.stop(); + if (omRatisSnapshotProvider != null) { + omRatisSnapshotProvider.stop(); } OMPerformanceMetrics.unregister(); } catch (Exception e) { @@ -3376,7 +3376,7 @@ public List getAcl(OzoneObj obj) throws IOException { * corresponding termIndex. Otherwise, return null. */ public TermIndex installSnapshotFromLeader(String leaderId) { - if (omSnapshotProvider == null) { + if (omRatisSnapshotProvider == null) { LOG.error("OM Snapshot Provider is not configured as there are no peer " + "nodes."); return null; @@ -3570,7 +3570,7 @@ private DBCheckpoint getDBCheckpointFromLeader(String leaderId) { "from the checkpoint.", leaderId); try { - return omSnapshotProvider.getOzoneManagerDBSnapshot(leaderId); + return omRatisSnapshotProvider.getOzoneManagerDBSnapshot(leaderId); } catch (IOException e) { LOG.error("Failed to download checkpoint from OM leader {}", leaderId, e); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java similarity index 85% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index 3ef2661f822..d79bf5512dd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.om.snapshot; +package org.apache.hadoop.ozone.om.ratis_snapshot; import java.io.File; import java.io.IOException; @@ -54,13 +54,25 @@ import org.slf4j.LoggerFactory; /** - * OzoneManagerSnapshotProvider downloads the latest checkpoint from the + * OmRatisSnapshotProvider downloads the latest checkpoint from the * leader OM and loads the checkpoint into State Machine. + * In addtion to the latest checkpoint, it also downloads any previous omSnapshots + * the leader has created. + * + * The term "snapshot" has two related but slightly different meanings in ozone. + * An "omSnapshot" is a copy of the om's metadata at a point in time. It is created + * by users through the "ozone sh snapshot create" cli. + * + * A "ratisSnapshot", (provided by this class), is used by om followers to bootstrap + * themselves to the current state of the om leader. ratisSnapshots will contain + * copies of all the individual "omSnapshot"s that exist on the leader at the time + * of the bootstrap. The follower needs these copies to respond the users snapshot + * requests when it becomes the leader. */ -public class OzoneManagerSnapshotProvider { +public class OmRatisSnapshotProvider { private static final Logger LOG = - LoggerFactory.getLogger(OzoneManagerSnapshotProvider.class); + LoggerFactory.getLogger(OmRatisSnapshotProvider.class); private final File omSnapshotDir; private Map peerNodesMap; @@ -68,10 +80,8 @@ public class OzoneManagerSnapshotProvider { private final boolean spnegoEnabled; private final URLConnectionFactory connectionFactory; - private static final String OM_SNAPSHOT_DB = "om.snapshot.db"; - - public OzoneManagerSnapshotProvider(MutableConfigurationSource conf, - File omRatisSnapshotDir, Map peerNodeDetails) { + public OmRatisSnapshotProvider(MutableConfigurationSource conf, + File omRatisSnapshotDir, Map peerNodeDetails) { LOG.info("Initializing OM Snapshot Provider"); this.omSnapshotDir = omRatisSnapshotDir; From 0c37119973b9ad3235d74bf840034efe3966b9bb Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 11 Nov 2022 16:52:50 -0800 Subject: [PATCH 040/120] checkstyle --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +- .../ozone/om/OmMetadataManagerImpl.java | 1 + .../OmRatisSnapshotProvider.java | 26 ++++++++++--------- .../ozone/om/snapshot/package-info.java | 23 ---------------- 4 files changed, 16 insertions(+), 36 deletions(-) delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/package-info.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 7bfd0d06cc3..af21b7507a6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -543,7 +543,7 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { @Test @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE", - "NP_NULL_ON_SOME_PATH"}) + "NP_NULL_ON_SOME_PATH"}) public void testOmSnapshots() throws Exception { // Get the leader OM String leaderOMNodeId = OmFailoverProxyUtil diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 7fe896aa5f8..780894d0516 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -333,6 +333,7 @@ public static OmMetadataManagerImpl createSnapshotMetadataManager( return smm; } + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) public static OmMetadataManagerImpl createCheckpointMetadataManager( OzoneConfiguration conf, DBCheckpoint checkpoint) throws IOException { Path path = checkpoint.getCheckpointLocation(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index d79bf5512dd..61d5999baff 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -55,19 +55,21 @@ /** * OmRatisSnapshotProvider downloads the latest checkpoint from the - * leader OM and loads the checkpoint into State Machine. - * In addtion to the latest checkpoint, it also downloads any previous omSnapshots - * the leader has created. + * leader OM and loads the checkpoint into State Machine. In addtion + * to the latest checkpoint, it also downloads any previous + * omSnapshots the leader has created. * - * The term "snapshot" has two related but slightly different meanings in ozone. - * An "omSnapshot" is a copy of the om's metadata at a point in time. It is created - * by users through the "ozone sh snapshot create" cli. + * The term "snapshot" has two related but slightly different meanings + * in ozone. An "omSnapshot" is a copy of the om's metadata at a + * point in time. It is created by users through the "ozone sh + * snapshot create" cli. * - * A "ratisSnapshot", (provided by this class), is used by om followers to bootstrap - * themselves to the current state of the om leader. ratisSnapshots will contain - * copies of all the individual "omSnapshot"s that exist on the leader at the time - * of the bootstrap. The follower needs these copies to respond the users snapshot - * requests when it becomes the leader. + * A "ratisSnapshot", (provided by this class), is used by om + * followers to bootstrap themselves to the current state of the om + * leader. ratisSnapshots will contain copies of all the individual + * "omSnapshot"s that exist on the leader at the time of the + * bootstrap. The follower needs these copies to respond the users + * snapshot requests when it becomes the leader. */ public class OmRatisSnapshotProvider { @@ -81,7 +83,7 @@ public class OmRatisSnapshotProvider { private final URLConnectionFactory connectionFactory; public OmRatisSnapshotProvider(MutableConfigurationSource conf, - File omRatisSnapshotDir, Map peerNodeDetails) { + File omRatisSnapshotDir, Map peerNodeDetails) { LOG.info("Initializing OM Snapshot Provider"); this.omSnapshotDir = omRatisSnapshotDir; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/package-info.java deleted file mode 100644 index 3c82a694078..00000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.snapshot; - -/** - * This package contains OM Ratis Snapshot related classes. - */ \ No newline at end of file From 2573caf03c57c2b391e5ffee98cbdd03f129d5f7 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 11 Nov 2022 18:39:41 -0800 Subject: [PATCH 041/120] added package info --- .../ozone/om/ratis_snapshot/package-info.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/package-info.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/package-info.java new file mode 100644 index 00000000000..20d0058c579 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.ratis_snapshot; + +/** + * This package contains OM Ratis Snapshot related classes. + */ \ No newline at end of file From 93fb5cfe0d934e6a77f952d0fd57ae9c88d09a58 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 11 Nov 2022 18:40:33 -0800 Subject: [PATCH 042/120] fixed import --- .../java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 780894d0516..1bd510f22b4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -32,6 +32,7 @@ import java.util.TreeSet; import java.util.stream.Collectors; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; From dc2b7b2224f85708900d2d46f0930a0924e9faea Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 11 Nov 2022 21:24:04 -0800 Subject: [PATCH 043/120] findbugs --- .../org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index d4695ec6f30..98242a07cba 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -377,7 +377,6 @@ public void testWriteDbDataWithoutOmSnapshot() FileUtil.unTar(tempFile, new File(testDirName)); Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); - int metaDirLength = metaDir.toString().length() + 1; Path finalCheckpointLocation = Paths.get(testDirName); From f07cfefaa66add9181b418e8445fd8ce9cd39454 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Sun, 13 Nov 2022 08:40:31 -0800 Subject: [PATCH 044/120] removed make dbDir --- .../java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 0e90f98b5c1..61a2b81865b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -138,9 +138,6 @@ public void testHardLinkCreation() throws IOException { OM_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); // Create dummy files to be linked to - if (!dbDir.mkdirs()) { - throw new IOException("failed to make directory: " + dbDir.toString()); - } if (!snapDir1.mkdirs()) { throw new IOException("failed to make directory: " + snapDir1.toString()); } From 43dadbb11b25e58b382329b2d37ac11692d6e24f Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 15 Nov 2022 13:06:08 -0800 Subject: [PATCH 045/120] restructured to pull in all of snapshot dir --- .../ozone/om/OMDBCheckpointServlet.java | 84 +++++++++---------- 1 file changed, 38 insertions(+), 46 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index b58697430fd..3046fa6abee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -18,9 +18,6 @@ package org.apache.hadoop.ozone.om; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; - import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; @@ -36,12 +33,13 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneConsts; - import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -50,20 +48,19 @@ import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; -import java.util.ArrayList; import java.util.Collection; -import java.util.LinkedHashSet; import java.util.HashMap; -import java.util.List; +import java.util.LinkedHashSet; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.om.OmSnapshotManager.createHardLinkList; -import static org.apache.hadoop.ozone.om.OmSnapshotManager.truncateFileName; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.truncateFileName; /** * Provides the current checkpoint Snapshot of the OM DB. (tar.gz) @@ -96,7 +93,7 @@ public void init() throws ServletException { return; } - OzoneConfiguration conf = om.getConfiguration(); + OzoneConfiguration conf = getConf(); // Only Ozone Admins and Recon are allowed Collection allowedUsers = new LinkedHashSet<>(om.getOmAdminUsernames()); @@ -123,32 +120,21 @@ private void getFilesForArchive(DBCheckpoint checkpoint, boolean includeSnapshotData) throws IOException, InterruptedException { Path dir = checkpoint.getCheckpointLocation(); - - try (Stream files = Files.list(dir)) { - for (Path file : files.collect(Collectors.toList())) { - // get the inode - Object key = Files.readAttributes( - file, BasicFileAttributes.class).fileKey(); - copyFiles.put(key, file); - } - } + processDir(dir, copyFiles, hardLinkFiles); if (!includeSnapshotData) { return; } - for (Path snapshotDir : getSnapshotDirs(checkpoint)) { - processDir(snapshotDir, copyFiles, hardLinkFiles); - } + waitForSnapshotDirs(checkpoint); + Path snapshotDir = Paths.get(OMStorage.getOmDbDir(getConf()).toString(), + OM_SNAPSHOT_DIR); + processDir(snapshotDir, copyFiles, hardLinkFiles); } - private List getSnapshotDirs(DBCheckpoint checkpoint) - throws IOException { - ArrayList list = new ArrayList<>(); - + private void waitForSnapshotDirs(DBCheckpoint checkpoint) + throws IOException, InterruptedException { // get snapshotInfo entries - OzoneConfiguration conf = ((OzoneManager) getServletContext() - .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) - .getConfiguration(); + OzoneConfiguration conf = getConf(); OmMetadataManagerImpl checkpointMetadataManager = OmMetadataManagerImpl.createCheckpointMetadataManager( @@ -157,14 +143,13 @@ private List getSnapshotDirs(DBCheckpoint checkpoint) iterator = checkpointMetadataManager .getSnapshotInfoTable().iterator()) { - // add each entries directory to the list + // wait for each directory while (iterator.hasNext()) { Table.KeyValue entry = iterator.next(); Path path = Paths.get(getSnapshotPath(conf, entry.getValue())); - list.add(path); + waitForDirToExist(path); } } - return list; } private void waitForDirToExist(Path dir) @@ -185,23 +170,29 @@ private void waitForDirToExist(Path dir) private void processDir(Path dir, Map copyFiles, Map hardLinkFiles) throws IOException, InterruptedException { - waitForDirToExist(dir); try (Stream files = Files.list(dir)) { for (Path file : files.collect(Collectors.toList())) { - // get the inode - Object key = Files.readAttributes( - file, BasicFileAttributes.class).fileKey(); - // If we already have the inode, store as hard link - if (copyFiles.containsKey(key)) { - hardLinkFiles.put(file, copyFiles.get(key)); - } else { - copyFiles.put(key, file); + if (file.toFile().isDirectory()) { + processDir(file, copyFiles, hardLinkFiles); + } else { + processFile(file, copyFiles, hardLinkFiles); } } } - } + private void processFile(Path file, Map copyFiles, + Map hardLinkFiles) throws IOException { + // get the inode + Object key = Files.readAttributes( + file, BasicFileAttributes.class).fileKey(); + // If we already have the inode, store as hard link + if (copyFiles.containsKey(key)) { + hardLinkFiles.put(file, copyFiles.get(key)); + } else { + copyFiles.put(key, file); + } + } @Override public void writeDbDataToStream(DBCheckpoint checkpoint, HttpServletRequest request, @@ -248,11 +239,7 @@ private void writeFilesToArchive(HashMap copyFiles, ArchiveOutputStream archiveOutputStream) throws IOException { - OzoneConfiguration conf = ((OzoneManager) getServletContext() - .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) - .getConfiguration(); - - File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); + File metaDirPath = ServerUtils.getOzoneMetaDirPath(getConf()); int truncateLength = metaDirPath.toString().length() + 1; for (Path file : copyFiles.values()) { @@ -271,4 +258,9 @@ private void writeFilesToArchive(HashMap copyFiles, } } + private OzoneConfiguration getConf() { + return ((OzoneManager) getServletContext() + .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) + .getConfiguration(); + } } From bf0511aa91d7d5bcb1503668b51f09349818c4b7 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 15 Nov 2022 13:20:38 -0800 Subject: [PATCH 046/120] added checkpoint state dir --- .../main/java/org/apache/hadoop/ozone/OzoneConsts.java | 5 +++++ .../java/org/apache/hadoop/hdds/utils/db/RDBStore.java | 4 ++-- .../org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java | 4 ++-- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 4 ++-- .../apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 4 ++-- .../apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 4 ++-- .../org/apache/hadoop/ozone/om/OmSnapshotManager.java | 4 ++-- .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 9 +++++---- .../apache/hadoop/ozone/om/TestOmSnapshotManager.java | 6 +++--- 9 files changed, 25 insertions(+), 19 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index e49a0a4a56a..907fa8d3710 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -555,6 +555,11 @@ private OzoneConsts() { public static final String OM_SNAPSHOT_NAME = "snapshotName"; public static final String OM_CHECKPOINT_DIR = "db.checkpoints"; public static final String OM_SNAPSHOT_DIR = "db.snapshots"; + public static final String OM_SNAPSHOT_CHECKPOINT_DIR = OM_SNAPSHOT_DIR + + OM_KEY_PREFIX + "checkpointState"; + public static final String OM_SNAPSHOT_DIFF_DIR = OM_SNAPSHOT_DIR + + OM_KEY_PREFIX + "diffState"; + public static final String OM_SNAPSHOT_INDICATOR = ".snapshot"; } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index e2049b45ec6..c2066b36d28 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -47,7 +47,7 @@ import org.slf4j.LoggerFactory; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; /** * RocksDB Store that supports creating Tables in DB. @@ -140,7 +140,7 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, //create snapshot directory if does not exist. snapshotsParentDir = Paths.get(dbLocation.getParent(), - OM_SNAPSHOT_DIR).toString(); + OM_SNAPSHOT_CHECKPOINT_DIR).toString(); File snapshotsDir = new File(snapshotsParentDir); if (!snapshotsDir.exists()) { boolean success = snapshotsDir.mkdir(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java index d90a1c6e038..ec1d17075fe 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java @@ -56,7 +56,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; /** * Tests Freon, with MiniOzoneCluster. @@ -112,7 +112,7 @@ public static void shutdown() { private String getDBCheckpointAbsolutePath(SnapshotInfo snapshotInfo) { return metaDir + OM_KEY_PREFIX + - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + + OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + OM_DB_NAME + snapshotInfo.getCheckpointDirName(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 98242a07cba..58b8212580d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -60,7 +60,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE; @@ -310,7 +310,7 @@ public void testWriteDbDataToStream() checkpointLocation.toString().length() + 1); Set finalCheckpointSet = getFiles(finalCheckpointLocation, testDirLength); - finalCheckpointSet.remove(OM_SNAPSHOT_DIR); + finalCheckpointSet.remove(OM_SNAPSHOT_CHECKPOINT_DIR); Assert.assertTrue("hardlink file exists in checkpoint dir", finalCheckpointSet.contains(OM_HARDLINK_FILE)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 3046fa6abee..e9236aa9514 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -56,7 +56,7 @@ import java.util.stream.Stream; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.om.OmSnapshotManager.createHardLinkList; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; @@ -127,7 +127,7 @@ private void getFilesForArchive(DBCheckpoint checkpoint, waitForSnapshotDirs(checkpoint); Path snapshotDir = Paths.get(OMStorage.getOmDbDir(getConf()).toString(), - OM_SNAPSHOT_DIR); + OM_SNAPSHOT_CHECKPOINT_DIR); processDir(snapshotDir, copyFiles, hardLinkFiles); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 1bd510f22b4..73d4ccfb02e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -103,7 +103,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import org.apache.ratis.util.ExitUtils; import org.eclipse.jetty.util.StringUtil; @@ -328,7 +328,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) public static OmMetadataManagerImpl createSnapshotMetadataManager( OzoneConfiguration conf, String snapshotDirName) throws IOException { File snapshotDir = new File(OMStorage.getOmDbDir(conf) + - OM_KEY_PREFIX + OM_SNAPSHOT_DIR); + OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR); OmMetadataManagerImpl smm = new OmMetadataManagerImpl(conf, snapshotDir, OM_DB_NAME + snapshotDirName); return smm; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 5484aff39fb..ee7969b4df8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -55,7 +55,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; @@ -237,7 +237,7 @@ public static String getSnapshotPrefix(String snapshotName) { public static String getSnapshotPath(OzoneConfiguration conf, SnapshotInfo snapshotInfo) { return OMStorage.getOmDbDir(conf) + - OM_KEY_PREFIX + OM_SNAPSHOT_DIR + OM_KEY_PREFIX + + OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + OM_DB_NAME + snapshotInfo.getCheckpointDirName(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 2ecab6c3af0..21ca468df5e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -231,7 +231,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY; import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE; import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.PREPARE_MARKER_KEY; import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT; @@ -3605,8 +3605,8 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, dbBackupDir); } File dbBackup = new File(dbBackupDir, oldDB.getName()); - File dbSnapshotsDir = new File(dbDir, OM_SNAPSHOT_DIR); - File dbSnapshotsBackup = new File(dbBackupDir, OM_SNAPSHOT_DIR); + File dbSnapshotsDir = new File(dbDir, OM_SNAPSHOT_CHECKPOINT_DIR); + File dbSnapshotsBackup = new File(dbBackupDir, OM_SNAPSHOT_CHECKPOINT_DIR); try { Files.move(oldDB.toPath(), dbBackup.toPath()); if (dbSnapshotsDir.exists()) { @@ -3654,7 +3654,8 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, // move the new snapshot directory into place and create hard links private void moveOmSnapshotData(Path dbPath, Path dbSnapshotsDir) throws IOException { - Path incomingSnapshotsDir = Paths.get(dbPath.toString(), OM_SNAPSHOT_DIR); + Path incomingSnapshotsDir = Paths.get(dbPath.toString(), + OM_SNAPSHOT_CHECKPOINT_DIR); if (incomingSnapshotsDir.toFile().exists()) { Files.move(incomingSnapshotsDir, dbSnapshotsDir); OmSnapshotManager.createHardLinks(dbPath); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 61a2b81865b..3ca3636c818 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -45,7 +45,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; import static org.mockito.Mockito.mock; @@ -131,9 +131,9 @@ public void testHardLinkCreation() throws IOException { File dbDir = new File(testDir.toString(), OM_DB_NAME); File snapDir1 = new File(testDir.toString(), - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + "dir1"); + OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); File snapDir2 = new File(testDir.toString(), - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + "dir2"); + OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir2"); File checkpointDir1 = new File(testDir.toString(), OM_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); From ee3457f108c1cf11b744ffd4e878139a3ef57129 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 15 Nov 2022 16:42:50 -0800 Subject: [PATCH 047/120] fixed up directories --- .../apache/hadoop/hdds/utils/db/RDBStore.java | 2 +- hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 4 ++ .../rocksdiff/RocksDBCheckpointDiffer.java | 13 +++--- .../ozone/om/TestOMDbCheckpointServlet.java | 43 +++++++++++++------ 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index c2066b36d28..b3ea51107c8 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -143,7 +143,7 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, OM_SNAPSHOT_CHECKPOINT_DIR).toString(); File snapshotsDir = new File(snapshotsParentDir); if (!snapshotsDir.exists()) { - boolean success = snapshotsDir.mkdir(); + boolean success = snapshotsDir.mkdirs(); if (!success) { throw new IOException( "Unable to create RocksDB snapshot directory: " + diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml index 0c03876df08..86661c51ac6 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml +++ b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml @@ -80,6 +80,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.ozone hdds-test-utils + + org.apache.ozone + hdds-common + diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 26884fb9fbc..6d3431c0aed 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -35,6 +35,7 @@ import org.rocksdb.TableProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.ozone.OzoneConsts; import java.io.BufferedWriter; import java.io.File; @@ -149,11 +150,12 @@ public RocksDBCheckpointDiffer(String metadataDir, String sstBackupDir, setCompactionLogDir(metadataDir, compactionLogDirName); - this.sstBackupDir = Paths.get(metadataDir, sstBackupDir) + "/"; + this.sstBackupDir = Paths.get(metadataDir, + OzoneConsts.OM_SNAPSHOT_DIFF_DIR, sstBackupDir) + "/"; // Create the directory if SST backup path does not already exist File dir = new File(this.sstBackupDir); - if (!dir.exists() && !dir.mkdir()) { + if (!dir.exists() && !dir.mkdirs()) { final String errorMsg = "Failed to create SST file backup directory. " + "Check if OM has write permission."; LOG.error(errorMsg); @@ -167,16 +169,17 @@ public RocksDBCheckpointDiffer(String metadataDir, String sstBackupDir, private void setCompactionLogDir(String metadataDir, String compactionLogDirName) { - final File parentDir = new File(metadataDir); + final File parentDir = new File(metadataDir, + OzoneConsts.OM_SNAPSHOT_DIFF_DIR); if (!parentDir.exists()) { - if (!parentDir.mkdir()) { + if (!parentDir.mkdirs()) { LOG.error("Error creating compaction log parent dir."); return; } } this.compactionLogDir = - Paths.get(metadataDir, compactionLogDirName).toString(); + Paths.get(parentDir.toString(), compactionLogDirName).toString(); File clDir = new File(compactionLogDir); if (!clDir.exists() && !clDir.mkdir()) { LOG.error("Error creating compaction log dir."); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 58b8212580d..8e6f4b97cb0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -59,8 +59,10 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; +import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE; @@ -297,20 +299,26 @@ public void testWriteDbDataToStream() // Untar the file into a temp folder to be examined String testDirName = folder.newFolder().getAbsolutePath(); int testDirLength = testDirName.length() + 1; - FileUtil.unTar(tempFile, new File(testDirName)); + String newDbDirName = testDirName + OM_KEY_PREFIX + OM_DB_NAME; + int newDbDirLength = newDbDirName.length() + 1; + File newDbDir = new File(newDbDirName); + newDbDir.mkdirs(); + FileUtil.unTar(tempFile, newDbDir); + // Move snapshot dir to correct location + new File(newDbDirName, OM_SNAPSHOT_DIR) + .renameTo(new File(newDbDir.getParent().toString(), OM_SNAPSHOT_DIR)); Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); int metaDirLength = metaDir.toString().length() + 1; Path finalCheckpointLocation = - Paths.get(testDirName); + Paths.get(newDbDirName); // Confirm the checkpoint directories match, (after remove extras) Set initialCheckpointSet = getFiles(checkpointLocation, checkpointLocation.toString().length() + 1); Set finalCheckpointSet = getFiles(finalCheckpointLocation, - testDirLength); - finalCheckpointSet.remove(OM_SNAPSHOT_CHECKPOINT_DIR); + newDbDirLength); Assert.assertTrue("hardlink file exists in checkpoint dir", finalCheckpointSet.contains(OM_HARDLINK_FILE)); @@ -328,15 +336,15 @@ public void testWriteDbDataToStream() Paths.get(finalSnapshotLocation.toString(), "CURRENT").toFile().exists()); - Set initialSnapshotSet = - getFiles(Paths.get(snapshotDirName), metaDirLength); - Set finalSnapshotSet = - getFiles(finalSnapshotLocation, testDirLength); + Set initialFullSet = + getFiles(Paths.get(metaDir.toString(), OM_SNAPSHOT_DIR), metaDirLength); + Set finalFullSet = + getFiles(Paths.get(testDirName, OM_SNAPSHOT_DIR), testDirLength); Assert.assertTrue("snapshot manifest found", - finalSnapshotSet.stream().anyMatch(s -> s.contains("MANIFEST"))); + finalFullSet.stream().anyMatch(s -> s.contains("MANIFEST"))); // check each line in the hard link file - Stream lines = Files.lines(Paths.get(testDirName, + Stream lines = Files.lines(Paths.get(newDbDirName, OM_HARDLINK_FILE)); boolean dummyLinkFound = false; for (String line: lines.collect(Collectors.toList())) { @@ -348,14 +356,14 @@ public void testWriteDbDataToStream() testDirName); } else { checkLine(shortSnapshotLocation, shortSnapshotLocation2, line); - if (line.startsWith(shortSnapshotLocation)) { - finalSnapshotSet.add(line.split("\t")[0]); + if (line.startsWith(OM_SNAPSHOT_DIR)) { + finalFullSet.add(line.split("\t")[0]); } } } Assert.assertTrue("dummy link found", dummyLinkFound); Assert.assertEquals("found expected snapshot files", - initialSnapshotSet, finalSnapshotSet); + initialFullSet, finalFullSet); } @Test @@ -376,6 +384,7 @@ public void testWriteDbDataWithoutOmSnapshot() int testDirLength = testDirName.length() + 1; FileUtil.unTar(tempFile, new File(testDirName)); + Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); Path finalCheckpointLocation = Paths.get(testDirName); @@ -395,9 +404,17 @@ public void testWriteDbDataWithoutOmSnapshot() private Set getFiles(Path path, int truncateLength) throws IOException { Set fileSet = new HashSet<>(); + return getFiles(path, truncateLength, fileSet); + } + + private Set getFiles(Path path, int truncateLength, Set fileSet) + throws IOException { try (Stream files = Files.list(path)) { for (Path file : files.collect(Collectors.toList())) { if (file != null) { + if (file.toFile().isDirectory()) { + getFiles(file, truncateLength, fileSet); + } if (!file.getFileName().toString().equals("dummyFile")) { fileSet.add(truncateFileName(truncateLength, file)); } From 0954bbaeacafd9f90f6bfd9844aacde2f5618e14 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 16 Nov 2022 15:50:30 -0800 Subject: [PATCH 048/120] restructured tarball to include all of snapshot dir --- .../org/apache/hadoop/ozone/OzoneConsts.java | 5 ++ .../apache/hadoop/hdds/utils/db/RDBStore.java | 16 +--- .../ozone/om/TestOMDbCheckpointServlet.java | 84 ++++++++++++------- .../ozone/om/OMDBCheckpointServlet.java | 3 +- 4 files changed, 64 insertions(+), 44 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 907fa8d3710..4bb2cde7fd8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -560,6 +560,11 @@ private OzoneConsts() { public static final String OM_SNAPSHOT_DIFF_DIR = OM_SNAPSHOT_DIR + OM_KEY_PREFIX + "diffState"; + // Name of the SST file backup directory placed under diff dir. + public static final String OM_COMPACTION_BACKUP_DIR = "compaction-sst-backup"; + // Name of the compaction log directory placed under diff dir. + public static final String OM_COMPACTION_LOG_DIR = "compaction-log"; + public static final String OM_SNAPSHOT_INDICATOR = ".snapshot"; } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index b3ea51107c8..415fdf51640 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -47,6 +47,8 @@ import org.slf4j.LoggerFactory; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_BACKUP_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_LOG_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; /** @@ -65,16 +67,6 @@ public class RDBStore implements DBStore { private final RDBMetrics rdbMetrics; private final RocksDBCheckpointDiffer rocksDBCheckpointDiffer; private final String dbJmxBeanName; - /** - * Name of the SST file backup directory placed under metadata dir. - * Can be made configurable later. - */ - private final String dbCompactionSSTBackupDirName = "compaction-sst-backup"; - /** - * Name of the compaction log directory placed under metadata dir. - * Can be made configurable later. - */ - private final String dbCompactionLogDirName = "compaction-log"; @VisibleForTesting public RDBStore(File dbFile, ManagedDBOptions options, @@ -100,8 +92,8 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, try { if (enableCompactionLog) { rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer( - dbLocation.getParent(), dbCompactionSSTBackupDirName, - dbCompactionLogDirName, dbLocation); + dbLocation.getParent(), OM_COMPACTION_BACKUP_DIR, + OM_COMPACTION_LOG_DIR, dbLocation); rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions); } else { rocksDBCheckpointDiffer = null; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 8e6f4b97cb0..3cdc11f21e0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -31,16 +31,19 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.security.Principal; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.google.common.collect.Sets; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -62,6 +65,8 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_BACKUP_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH; @@ -105,6 +110,7 @@ public class TestOMDbCheckpointServlet { private File metaDir; private String snapshotDirName; private String snapshotDirName2; + private Path compactionDirPath; private DBCheckpoint dbCheckpoint; @Rule @@ -143,6 +149,7 @@ public void write(int b) throws IOException { fileOutputStream.write(b); } }; + } /** @@ -329,6 +336,8 @@ public void testWriteDbDataToStream() truncateFileName(metaDirLength, Paths.get(snapshotDirName)); String shortSnapshotLocation2 = truncateFileName(metaDirLength, Paths.get(snapshotDirName2)); + String shortCompactionDirLocation = + truncateFileName(metaDirLength, compactionDirPath); Path finalSnapshotLocation = Paths.get(testDirName, shortSnapshotLocation); @@ -346,22 +355,20 @@ public void testWriteDbDataToStream() // check each line in the hard link file Stream lines = Files.lines(Paths.get(newDbDirName, OM_HARDLINK_FILE)); - boolean dummyLinkFound = false; + List dummyLinkLines = new ArrayList<>(); for (String line: lines.collect(Collectors.toList())) { Assert.assertFalse("CURRENT file is not a hard link", line.contains("CURRENT")); if (line.contains("dummyFile")) { - dummyLinkFound = true; - checkDummyFile(shortSnapshotLocation, shortSnapshotLocation2, line, - testDirName); + dummyLinkLines.add(line); } else { checkLine(shortSnapshotLocation, shortSnapshotLocation2, line); - if (line.startsWith(OM_SNAPSHOT_DIR)) { - finalFullSet.add(line.split("\t")[0]); - } + finalFullSet.add(line.split("\t")[0]); } } - Assert.assertTrue("dummy link found", dummyLinkFound); + Set directories = Sets.newHashSet( + shortSnapshotLocation, shortSnapshotLocation2, shortCompactionDirLocation); + checkDummyFile(directories, dummyLinkLines, testDirName); Assert.assertEquals("found expected snapshot files", initialFullSet, finalFullSet); } @@ -427,29 +434,36 @@ private Set getFiles(Path path, int truncateLength, Set fileSet) // tests to see that dummy entry in hardlink file looks something like: // "dir1/dummyFile dir2/dummyFile" @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) - private void checkDummyFile(String dir0, String dir1, String line, + private void checkDummyFile(Set directories, List lines, String testDirName) { - String[] files = line.split("\t"); - Assert.assertTrue("dummy entry contains first directory", - files[0].startsWith(dir0) || files[1].startsWith(dir0)); - Assert.assertTrue("dummy entry contains second directory", - files[0].startsWith(dir1) || files[1].startsWith(dir1)); - Path path0 = Paths.get(files[0]); - Path path1 = Paths.get(files[1]); - Assert.assertNotEquals("dummy entry parent directories differ", - path0.getParent(), path1.getParent()); - Assert.assertTrue("dummy entries contains dummyFile name", - path0.getFileName().toString().equals("dummyFile") && - path1.getFileName().toString().equals("dummyFile")); - int dummyFileCount = 0; - if (Paths.get(testDirName, dir0, "dummyFile").toFile().exists()) { - dummyFileCount++; - } - if (Paths.get(testDirName, dir1, "dummyFile").toFile().exists()) { - dummyFileCount++; + // find the real file + String realDir = null; + for (String dir: directories) { + if (Paths.get(testDirName, dir, "dummyFile").toFile().exists()) { + directories.remove(dir); + realDir = dir; + } + } + + Assert.assertEquals("Exactly one copy of the file existed in the tarball", + directories.size(), 2); + + String dir0 = (String)directories.toArray()[0]; + String dir1 = (String)directories.toArray()[1]; + Assert.assertNotEquals("link directories are different", dir0, dir1); + + for (String line : lines) { + String[] files = line.split("\t"); + Assert.assertTrue("dummy entry contains first directory", + files[0].startsWith(dir0) || files[0].startsWith(dir1)); + Assert.assertTrue("dummy entry contains real dir", + files[1].startsWith(realDir)); + Path path0 = Paths.get(files[0]); + Path path1 = Paths.get(files[1]); + Assert.assertTrue("dummy entries contains dummyFile name", + path0.getFileName().toString().equals("dummyFile") && + path1.getFileName().toString().equals("dummyFile")); } - Assert.assertEquals("exactly one dummy file exists in the output dir", - dummyFileCount, 1); } // tests line in hard link file looks something like: @@ -510,14 +524,22 @@ private void prepArchiveData() throws Exception { // they are not in the checkpoint directory Path dummyFile = Paths.get(snapshotDirName, "dummyFile"); Path dummyLink = Paths.get(snapshotDirName2, "dummyFile"); + + compactionDirPath = Paths.get(metaDir.toString(), + OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_BACKUP_DIR); + Path dummyLink2 = Paths.get(compactionDirPath.toString(), "dummyFile"); Files.write(dummyFile, "dummyData".getBytes(StandardCharsets.UTF_8)); Files.createLink(dummyLink, dummyFile); + Files.createLink(dummyLink2, dummyFile); + Path currentFile = Paths.get(folder.newFolder().getAbsolutePath(), + OM_DB_NAME, "CURRENT"); + Path currentLink = Paths.get(compactionDirPath.toString(), "CURRENT"); + Files.createLink(currentLink, currentFile); dbCheckpoint = cluster.getOzoneManager() .getMetadataManager().getStore() .getCheckpoint(true); - } - + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index e9236aa9514..602a7b38fad 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -57,6 +57,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.om.OmSnapshotManager.createHardLinkList; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; @@ -127,7 +128,7 @@ private void getFilesForArchive(DBCheckpoint checkpoint, waitForSnapshotDirs(checkpoint); Path snapshotDir = Paths.get(OMStorage.getOmDbDir(getConf()).toString(), - OM_SNAPSHOT_CHECKPOINT_DIR); + OM_SNAPSHOT_DIR); processDir(snapshotDir, copyFiles, hardLinkFiles); } From 1cd52754c421fe92f7de14c91d5e70024764bc7b Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 16 Nov 2022 16:04:33 -0800 Subject: [PATCH 049/120] cleanup --- .../org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 3cdc11f21e0..4073f2820c7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -531,7 +531,7 @@ private void prepArchiveData() throws Exception { Files.write(dummyFile, "dummyData".getBytes(StandardCharsets.UTF_8)); Files.createLink(dummyLink, dummyFile); Files.createLink(dummyLink2, dummyFile); - Path currentFile = Paths.get(folder.newFolder().getAbsolutePath(), + Path currentFile = Paths.get(metaDir.toString(), OM_DB_NAME, "CURRENT"); Path currentLink = Paths.get(compactionDirPath.toString(), "CURRENT"); Files.createLink(currentLink, currentFile); From d229f28f07156bf73f689912109b78f7327b3179 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 17 Nov 2022 07:48:09 -0800 Subject: [PATCH 050/120] cleanup --- .../ozone/om/TestOMDbCheckpointServlet.java | 26 +++---- .../ozone/om/OMDBCheckpointServlet.java | 76 +++++++++++-------- .../hadoop/ozone/om/OmSnapshotManager.java | 76 +++++++++++-------- .../apache/hadoop/ozone/om/OzoneManager.java | 7 +- 4 files changed, 103 insertions(+), 82 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 4073f2820c7..586f09e0687 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.HashSet; import java.util.List; @@ -64,7 +65,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_BACKUP_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; @@ -149,7 +149,6 @@ public void write(int b) throws IOException { fileOutputStream.write(b); } }; - } /** @@ -436,20 +435,19 @@ private Set getFiles(Path path, int truncateLength, Set fileSet) @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private void checkDummyFile(Set directories, List lines, String testDirName) { - // find the real file - String realDir = null; - for (String dir: directories) { - if (Paths.get(testDirName, dir, "dummyFile").toFile().exists()) { - directories.remove(dir); - realDir = dir; - } + // find the real file + String realDir = null; + for (String dir: directories) { + if (Paths.get(testDirName, dir, "dummyFile").toFile().exists()) { + Assert.assertNull("Exactly one copy of the file existed in the tarball", realDir); + realDir = dir; } + } - Assert.assertEquals("Exactly one copy of the file existed in the tarball", - directories.size(), 2); - - String dir0 = (String)directories.toArray()[0]; - String dir1 = (String)directories.toArray()[1]; + directories.remove(realDir); + Iterator directoryIterator = directories.iterator(); + String dir0 = directoryIterator.next(); + String dir1 = directoryIterator.next(); Assert.assertNotEquals("link directories are different", dir0, dir1); for (String line : lines) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 602a7b38fad..d2a2b0ba0a4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -115,28 +115,66 @@ public void init() throws ServletException { om.isSpnegoEnabled()); } + @Override + public void writeDbDataToStream(DBCheckpoint checkpoint, + HttpServletRequest request, + OutputStream destination) + throws IOException, InterruptedException, CompressorException { + // Map of inodes to path + HashMap copyFiles = new HashMap<>(); + // Map of link to path + HashMap hardLinkFiles = new HashMap<>(); + + getFilesForArchive(checkpoint, copyFiles, hardLinkFiles, + includeSnapshotData(request)); + + try (CompressorOutputStream gzippedOut = new CompressorStreamFactory() + .createCompressorOutputStream(CompressorStreamFactory.GZIP, + destination)) { + + try (TarArchiveOutputStream archiveOutputStream = + new TarArchiveOutputStream(gzippedOut)) { + archiveOutputStream + .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + + writeFilesToArchive(copyFiles, hardLinkFiles, archiveOutputStream); + } + } catch (CompressorException e) { + throw new IOException( + "Can't compress the checkpoint: " + + checkpoint.getCheckpointLocation(), e); + } + } + private void getFilesForArchive(DBCheckpoint checkpoint, Map copyFiles, Map hardLinkFiles, boolean includeSnapshotData) throws IOException, InterruptedException { + + // Get the active fs files Path dir = checkpoint.getCheckpointLocation(); processDir(dir, copyFiles, hardLinkFiles); + if (!includeSnapshotData) { return; } + // Get the snapshot files waitForSnapshotDirs(checkpoint); Path snapshotDir = Paths.get(OMStorage.getOmDbDir(getConf()).toString(), OM_SNAPSHOT_DIR); processDir(snapshotDir, copyFiles, hardLinkFiles); } + // The snapshotInfo table may contain a snapshot that + // doesn't yet exist on the fs, so wait a few seconds for it private void waitForSnapshotDirs(DBCheckpoint checkpoint) throws IOException, InterruptedException { - // get snapshotInfo entries + OzoneConfiguration conf = getConf(); + // get snapshotInfo entries OmMetadataManagerImpl checkpointMetadataManager = OmMetadataManagerImpl.createCheckpointMetadataManager( conf, checkpoint); @@ -144,7 +182,7 @@ private void waitForSnapshotDirs(DBCheckpoint checkpoint) iterator = checkpointMetadataManager .getSnapshotInfoTable().iterator()) { - // wait for each directory + // for each entry, wait for corresponding directory while (iterator.hasNext()) { Table.KeyValue entry = iterator.next(); Path path = Paths.get(getSnapshotPath(conf, entry.getValue())); @@ -194,37 +232,8 @@ private void processFile(Path file, Map copyFiles, copyFiles.put(key, file); } } - @Override - public void writeDbDataToStream(DBCheckpoint checkpoint, - HttpServletRequest request, - OutputStream destination) - throws IOException, InterruptedException, CompressorException { - // Map of inodes to path - HashMap copyFiles = new HashMap<>(); - // Map of link to path - HashMap hardLinkFiles = new HashMap<>(); - - getFilesForArchive(checkpoint, copyFiles, hardLinkFiles, - includeSnapshotData(request)); - - try (CompressorOutputStream gzippedOut = new CompressorStreamFactory() - .createCompressorOutputStream(CompressorStreamFactory.GZIP, - destination)) { - - try (TarArchiveOutputStream archiveOutputStream = - new TarArchiveOutputStream(gzippedOut)) { - archiveOutputStream - .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - - writeFilesToArchive(copyFiles, hardLinkFiles, archiveOutputStream); - } - } catch (CompressorException e) { - throw new IOException( - "Can't compress the checkpoint: " + - checkpoint.getCheckpointLocation(), e); - } - } + // returns value of http request parameter private boolean includeSnapshotData(HttpServletRequest request) { String includeParam = request.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA); @@ -243,6 +252,7 @@ private void writeFilesToArchive(HashMap copyFiles, File metaDirPath = ServerUtils.getOzoneMetaDirPath(getConf()); int truncateLength = metaDirPath.toString().length() + 1; + // Go through each of the files to be copied and add to archive for (Path file : copyFiles.values()) { String fixedFile = truncateFileName(truncateLength, file); if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { @@ -252,6 +262,8 @@ private void writeFilesToArchive(HashMap copyFiles, includeFile(file.toFile(), fixedFile, archiveOutputStream); } + + // Create list of hard links if (!hardLinkFiles.isEmpty()) { Path hardLinkFile = createHardLinkList(truncateLength, hardLinkFiles); includeFile(hardLinkFile.toFile(), OmSnapshotManager.OM_HARDLINK_FILE, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index ee7969b4df8..72322ae95c2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -130,7 +130,7 @@ public OmSnapshot load(@Nonnull String snapshotTableKey) = notification -> { try { // close snapshot's rocksdb on eviction - Objects.requireNonNull(notification.getValue()).close(); + notification.getValue().close(); } catch (IOException e) { LOG.error("Failed to close snapshot: {} {}", notification.getKey(), e); @@ -280,6 +280,43 @@ private void verifySnapshotInfoForSnapDiff(final SnapshotInfo fromSnapshot, throw new IOException("fromSnapshot:" + fromSnapshot.getName() + " should be older than to toSnapshot:" + toSnapshot.getName()); } + + /** + * Create file of links to add to tarball + * Format of entries are either: + * dir1/fileTo fileFrom + * for files in active db or: + * dir1/fileTo dir2/fileFrom + * for files in another directory, (either another snapshot dir or + * sst compaction backup directory) + * @param truncateLength length of initial path to trim in file path + * @param hardLinkFiles a map of link->file paths + * @return Path to the file of links created + */ + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) + static Path createHardLinkList(int truncateLength, + Map hardLinkFiles) + throws IOException { + Path data = Files.createTempFile("data", "txt"); + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : hardLinkFiles.entrySet()) { + String fixedFile = truncateFileName(truncateLength, entry.getValue()); + // If this file is from the active db, strip the path + if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { + fixedFile = Paths.get(fixedFile).getFileName().toString(); + } + sb.append(truncateFileName(truncateLength, entry.getKey())) + .append("\t") + .append(fixedFile) + .append("\n"); + } + Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); + return data; + } + + // get the filename without the introductory metadata directory + static String truncateFileName(int truncateLength, Path file) { + return file.toString().substring(truncateLength); } // Create hard links listed in OM_HARDLINK_FILE @@ -287,8 +324,11 @@ static void createHardLinks(Path dbPath) throws IOException { File hardLinkFile = new File(dbPath.toString(), OM_HARDLINK_FILE); if (hardLinkFile.exists()) { + // Read file List lines = Files.lines(hardLinkFile.toPath()).collect(Collectors.toList()); + + // Create a link for each line for (String l : lines) { String from = l.split("\t")[1]; String to = l.split("\t")[0]; @@ -303,7 +343,8 @@ static void createHardLinks(Path dbPath) throws IOException { } } - // Get the full path of OM_HARDLINK_FILE entry + + // Prepend the full path to the hard link entry entry @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private static Path getFullPath(Path dbPath, String fileName) { File file = new File(fileName); @@ -315,35 +356,4 @@ private static Path getFullPath(Path dbPath, String fileName) { return Paths.get(dbPath.getParent().toString(), fileName); } - // Create list of links to add to follower - // Format of entries are either: - // db.snapshot/dir1/fileTo fileFrom - // for files in active db or: - // db.snapshot/dir1/fileTo parent/dir2/fileFrom - // for files in another directory, (either another snapshot dir or - // sst compaction backup directory) - @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) - static Path createHardLinkList(int truncateLength, - Map hardLinkFiles) - throws IOException { - Path data = Files.createTempFile("data", "txt"); - StringBuilder sb = new StringBuilder(); - for (Map.Entry entry : hardLinkFiles.entrySet()) { - String fixedFile = truncateFileName(truncateLength, entry.getValue()); - // If this file is from the active db, strip the path - if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { - fixedFile = Paths.get(fixedFile).getFileName().toString(); - } - sb.append(truncateFileName(truncateLength, entry.getKey())) - .append("\t") - .append(fixedFile) - .append("\n"); - } - Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); - return data; - } - - static String truncateFileName(int truncateLength, Path file) { - return file.toString().substring(truncateLength); - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 21ca468df5e..27935c6d5a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -232,6 +232,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE; import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.PREPARE_MARKER_KEY; import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT; @@ -3605,8 +3606,8 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, dbBackupDir); } File dbBackup = new File(dbBackupDir, oldDB.getName()); - File dbSnapshotsDir = new File(dbDir, OM_SNAPSHOT_CHECKPOINT_DIR); - File dbSnapshotsBackup = new File(dbBackupDir, OM_SNAPSHOT_CHECKPOINT_DIR); + File dbSnapshotsDir = new File(dbDir, OM_SNAPSHOT_DIR); + File dbSnapshotsBackup = new File(dbBackupDir, OM_SNAPSHOT_DIR); try { Files.move(oldDB.toPath(), dbBackup.toPath()); if (dbSnapshotsDir.exists()) { @@ -3655,7 +3656,7 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, private void moveOmSnapshotData(Path dbPath, Path dbSnapshotsDir) throws IOException { Path incomingSnapshotsDir = Paths.get(dbPath.toString(), - OM_SNAPSHOT_CHECKPOINT_DIR); + OM_SNAPSHOT_DIR); if (incomingSnapshotsDir.toFile().exists()) { Files.move(incomingSnapshotsDir, dbSnapshotsDir); OmSnapshotManager.createHardLinks(dbPath); From 7f175ee9b0de73837155652a2f57ce383b594196 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 17 Nov 2022 11:14:44 -0800 Subject: [PATCH 051/120] cleanup hardlink test --- .../ozone/om/OMDBCheckpointServlet.java | 5 +-- .../hadoop/ozone/om/OmSnapshotManager.java | 9 ++++++ .../ozone/om/TestOmSnapshotManager.java | 32 ++++++++++++------- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index d2a2b0ba0a4..9420d3b239c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -46,7 +46,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.util.Collection; import java.util.HashMap; @@ -56,7 +55,6 @@ import java.util.stream.Stream; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.om.OmSnapshotManager.createHardLinkList; @@ -223,8 +221,7 @@ private void processDir(Path dir, Map copyFiles, private void processFile(Path file, Map copyFiles, Map hardLinkFiles) throws IOException { // get the inode - Object key = Files.readAttributes( - file, BasicFileAttributes.class).fileKey(); + Object key = OmSnapshotManager.getINode(file); // If we already have the inode, store as hard link if (copyFiles.containsKey(key)) { hardLinkFiles.put(file, copyFiles.get(key)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 72322ae95c2..18a31955b58 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om; +import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -46,6 +47,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; import java.util.List; import java.util.Map; import java.util.Objects; @@ -176,6 +178,13 @@ public static DBCheckpoint createOmSnapshotCheckpoint( return dbCheckpoint; } + @VisibleForTesting + static Object getINode(Path file) throws IOException { + Object key = Files.readAttributes( + file, BasicFileAttributes.class).fileKey(); + return key; + } + // Get OmSnapshot if the keyname has ".snapshot" key indicator public IOmMetadataReader checkForSnapshot(String volumeName, String bucketName, String keyname) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 3ca3636c818..f9ecd7cc5c0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -47,6 +47,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getINode; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; @@ -129,25 +130,27 @@ public void testCloseOnEviction() throws IOException { public void testHardLinkCreation() throws IOException { byte[] dummyData = {0}; - File dbDir = new File(testDir.toString(), OM_DB_NAME); - File snapDir1 = new File(testDir.toString(), - OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); - File snapDir2 = new File(testDir.toString(), - OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir2"); - File checkpointDir1 = new File(testDir.toString(), - OM_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); // Create dummy files to be linked to + File snapDir1 = new File(testDir.toString(), + OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); if (!snapDir1.mkdirs()) { throw new IOException("failed to make directory: " + snapDir1.toString()); } + Files.write(Paths.get(snapDir1.toString(), "s1"), dummyData); + + File snapDir2 = new File(testDir.toString(), + OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir2"); if (!snapDir2.mkdirs()) { - throw new IOException("failed to make directory: " + dbDir.toString()); + throw new IOException("failed to make directory: " + snapDir2.toString()); } + + File dbDir = new File(testDir.toString(), OM_DB_NAME); Files.write(Paths.get(dbDir.toString(), "f1"), dummyData); - Files.write(Paths.get(snapDir1.toString(), "s1"), dummyData); // Create map of links to dummy files + File checkpointDir1 = new File(testDir.toString(), + OM_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); Map hardLinkFiles = new HashMap<>(); hardLinkFiles.put(Paths.get(snapDir2.toString(), "f1"), Paths.get(checkpointDir1.toString(), "f1")); @@ -164,8 +167,15 @@ public void testHardLinkCreation() throws IOException { OmSnapshotManager.createHardLinks(dbDir.toPath()); // Confirm expected links - for (Path path : hardLinkFiles.keySet()) { - Assert.assertTrue(path.toFile().exists()); + for (Map.Entry entry : hardLinkFiles.entrySet()) { + Assert.assertTrue(entry.getKey().toFile().exists()); + Path value = entry.getValue(); + // convert checkpoint path to om.db + if (value.toString().contains(OM_CHECKPOINT_DIR)) { + value = Paths.get(dbDir.toString(), value.getFileName().toString()); + } + Assert.assertEquals("link matches original file", + getINode(entry.getKey()), getINode(value)); } } From 793cea29f2e9dca50ee17c14f62932e81ddaa7a8 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 17 Nov 2022 11:26:38 -0800 Subject: [PATCH 052/120] rebase cleanup --- .../main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 18a31955b58..d113c6150fe 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -289,7 +289,7 @@ private void verifySnapshotInfoForSnapDiff(final SnapshotInfo fromSnapshot, throw new IOException("fromSnapshot:" + fromSnapshot.getName() + " should be older than to toSnapshot:" + toSnapshot.getName()); } - + } /** * Create file of links to add to tarball * Format of entries are either: From 75beaf6da868db5e68d661efae25c5519ae862b9 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 17 Nov 2022 14:52:34 -0800 Subject: [PATCH 053/120] cleanup --- .../ozone/om/TestOMDbCheckpointServlet.java | 220 +++++++++--------- 1 file changed, 112 insertions(+), 108 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 586f09e0687..85610a86389 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -293,10 +293,12 @@ public void testSpnegoEnabled() throws Exception { @Test public void testWriteDbDataToStream() throws Exception { - prepArchiveData(); - // Do include snapshot data + prepSnapshotData(); + // set http param to include snapshot data when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)) .thenReturn("true"); + + // get the tarball try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock, fileOutputStream); @@ -310,19 +312,17 @@ public void testWriteDbDataToStream() File newDbDir = new File(newDbDirName); newDbDir.mkdirs(); FileUtil.unTar(tempFile, newDbDir); + // Move snapshot dir to correct location new File(newDbDirName, OM_SNAPSHOT_DIR) .renameTo(new File(newDbDir.getParent().toString(), OM_SNAPSHOT_DIR)); - Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); - int metaDirLength = metaDir.toString().length() + 1; - Path finalCheckpointLocation = - Paths.get(newDbDirName); - // Confirm the checkpoint directories match, (after remove extras) + Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); Set initialCheckpointSet = getFiles(checkpointLocation, checkpointLocation.toString().length() + 1); + Path finalCheckpointLocation = Paths.get(newDbDirName); Set finalCheckpointSet = getFiles(finalCheckpointLocation, newDbDirLength); @@ -331,6 +331,7 @@ public void testWriteDbDataToStream() finalCheckpointSet.remove(OM_HARDLINK_FILE); Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); + int metaDirLength = metaDir.toString().length() + 1; String shortSnapshotLocation = truncateFileName(metaDirLength, Paths.get(snapshotDirName)); String shortSnapshotLocation2 = @@ -340,34 +341,32 @@ public void testWriteDbDataToStream() Path finalSnapshotLocation = Paths.get(testDirName, shortSnapshotLocation); - Assert.assertTrue("CURRENT file exists", - Paths.get(finalSnapshotLocation.toString(), - "CURRENT").toFile().exists()); - - Set initialFullSet = - getFiles(Paths.get(metaDir.toString(), OM_SNAPSHOT_DIR), metaDirLength); Set finalFullSet = getFiles(Paths.get(testDirName, OM_SNAPSHOT_DIR), testDirLength); - Assert.assertTrue("snapshot manifest found", - finalFullSet.stream().anyMatch(s -> s.contains("MANIFEST"))); // check each line in the hard link file Stream lines = Files.lines(Paths.get(newDbDirName, OM_HARDLINK_FILE)); - List dummyLinkLines = new ArrayList<>(); + + List fabricatedLinkLines = new ArrayList<>(); for (String line: lines.collect(Collectors.toList())) { Assert.assertFalse("CURRENT file is not a hard link", line.contains("CURRENT")); - if (line.contains("dummyFile")) { - dummyLinkLines.add(line); + if (line.contains("fabricatedFile")) { + fabricatedLinkLines.add(line); } else { checkLine(shortSnapshotLocation, shortSnapshotLocation2, line); + // add links to the final set finalFullSet.add(line.split("\t")[0]); } } + Set directories = Sets.newHashSet( shortSnapshotLocation, shortSnapshotLocation2, shortCompactionDirLocation); - checkDummyFile(directories, dummyLinkLines, testDirName); + checkFabricatedLines(directories, fabricatedLinkLines, testDirName); + + Set initialFullSet = + getFiles(Paths.get(metaDir.toString(), OM_SNAPSHOT_DIR), metaDirLength); Assert.assertEquals("found expected snapshot files", initialFullSet, finalFullSet); } @@ -375,11 +374,13 @@ public void testWriteDbDataToStream() @Test public void testWriteDbDataWithoutOmSnapshot() throws Exception { - prepArchiveData(); + prepSnapshotData(); - // Don't include snapshot data + // set http param to exclude snapshot data when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)) .thenReturn(null); + + // get the tarball try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock, fileOutputStream); @@ -390,14 +391,11 @@ public void testWriteDbDataWithoutOmSnapshot() int testDirLength = testDirName.length() + 1; FileUtil.unTar(tempFile, new File(testDirName)); - + // Confirm the checkpoint directories match Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); - Path finalCheckpointLocation = - Paths.get(testDirName); - - // Confirm the checkpoint directories match, (after remove extras) Set initialCheckpointSet = getFiles(checkpointLocation, checkpointLocation.toString().length() + 1); + Path finalCheckpointLocation = Paths.get(testDirName); Set finalCheckpointSet = getFiles(finalCheckpointLocation, testDirLength); @@ -406,40 +404,107 @@ public void testWriteDbDataWithoutOmSnapshot() - @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) + private void prepSnapshotData() throws Exception { + setupCluster(); + metaDir = OMStorage.getOmDbDir(conf); + + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); + + // Create dummy keys for snapshotting + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + "content"); + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + "content"); + + // this sleep can be removed after this is fixed: + // https://issues.apache.org/jira/browse/HDDS-7279 + Thread.sleep(2000); + snapshotDirName = + createSnapshot(bucket.getVolumeName(), bucket.getName()); + snapshotDirName2 = + createSnapshot(bucket.getVolumeName(), bucket.getName()); + + + // create fabricated links to snapshot dirs + // to confirm that links are recognized even if + // they are don't point to the checkpoint directory + Path fabricatedFile = Paths.get(snapshotDirName, "fabricatedFile"); + Path fabricatedLink = Paths.get(snapshotDirName2, "fabricatedFile"); + + Files.write(fabricatedFile, "fabricatedData".getBytes(StandardCharsets.UTF_8)); + Files.createLink(fabricatedLink, fabricatedFile); + + // simulate links from the compaction dir + compactionDirPath = Paths.get(metaDir.toString(), + OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_BACKUP_DIR); + Path fabricatedLink2 = Paths.get(compactionDirPath.toString(), "fabricatedFile"); + Files.createLink(fabricatedLink2, fabricatedFile); + Path currentFile = Paths.get(metaDir.toString(), + OM_DB_NAME, "CURRENT"); + Path currentLink = Paths.get(compactionDirPath.toString(), "CURRENT"); + Files.createLink(currentLink, currentFile); + + dbCheckpoint = cluster.getOzoneManager() + .getMetadataManager().getStore() + .getCheckpoint(true); + + } + + private String createSnapshot(String vname, String bname) + throws IOException, InterruptedException, TimeoutException { + final OzoneManager om = cluster.getOzoneManager(); + String snapshotName = UUID.randomUUID().toString(); + OzoneManagerProtocol writeClient = cluster.getRpcClient().getObjectStore() + .getClientProxy().getOzoneManagerClient(); + + writeClient.createSnapshot(vname, bname, snapshotName); + SnapshotInfo snapshotInfo = om.getMetadataManager().getSnapshotInfoTable() + .get(SnapshotInfo.getTableKey(vname, bname, snapshotName)); + String snapshotPath = getSnapshotPath(conf, snapshotInfo) + + OM_KEY_PREFIX; + GenericTestUtils.waitFor(() -> new File(snapshotPath).exists(), + 100, 2000); + return snapshotPath; + } + private Set getFiles(Path path, int truncateLength) throws IOException { - Set fileSet = new HashSet<>(); - return getFiles(path, truncateLength, fileSet); + return getFiles(path, truncateLength, new HashSet<>()); } + // Get all files below path, recursively, (skipping fabricated files) + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private Set getFiles(Path path, int truncateLength, Set fileSet) throws IOException { try (Stream files = Files.list(path)) { for (Path file : files.collect(Collectors.toList())) { - if (file != null) { - if (file.toFile().isDirectory()) { - getFiles(file, truncateLength, fileSet); - } - if (!file.getFileName().toString().equals("dummyFile")) { - fileSet.add(truncateFileName(truncateLength, file)); - } + if (file.toFile().isDirectory()) { + getFiles(file, truncateLength, fileSet); + } + if (!file.getFileName().toString().equals("fabricatedFile")) { + fileSet.add(truncateFileName(truncateLength, file)); } } } return fileSet; } - // tests to see that dummy entry in hardlink file looks something like: - // "dir1/dummyFile dir2/dummyFile" + // tests to see that fabricated link lines in hardlink file are properly formatted + // "dir1/fabricatedFile dir2/fabricatedFile" + // + // The "fabricated" files/links are ones I've created by hand to fully test the + // code, (as opposed to the "natural" files/links created by the + // create snapshot process.) @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) - private void checkDummyFile(Set directories, List lines, + private void checkFabricatedLines(Set directories, List lines, String testDirName) { // find the real file String realDir = null; for (String dir: directories) { - if (Paths.get(testDirName, dir, "dummyFile").toFile().exists()) { - Assert.assertNull("Exactly one copy of the file existed in the tarball", realDir); + if (Paths.get(testDirName, dir, "fabricatedFile").toFile().exists()) { + Assert.assertNull( + "Exactly one copy of the fabricated file exists in the tarball", + realDir); realDir = dir; } } @@ -452,19 +517,19 @@ private void checkDummyFile(Set directories, List lines, for (String line : lines) { String[] files = line.split("\t"); - Assert.assertTrue("dummy entry contains first directory", + Assert.assertTrue("fabricated entry contains valid first directory", files[0].startsWith(dir0) || files[0].startsWith(dir1)); - Assert.assertTrue("dummy entry contains real dir", + Assert.assertTrue("fabricated entry contains correct real directory", files[1].startsWith(realDir)); Path path0 = Paths.get(files[0]); Path path1 = Paths.get(files[1]); - Assert.assertTrue("dummy entries contains dummyFile name", - path0.getFileName().toString().equals("dummyFile") && - path1.getFileName().toString().equals("dummyFile")); + Assert.assertTrue("fabricated entries contains correct file name", + path0.getFileName().toString().equals("fabricatedFile") && + path1.getFileName().toString().equals("fabricatedFile")); } } - // tests line in hard link file looks something like: + // validates line in hard link file. should look something like: // "dir1/x.sst x.sst" private void checkLine(String shortSnapshotLocation, String shortSnapshotLocation2, @@ -479,65 +544,4 @@ private void checkLine(String shortSnapshotLocation, Assert.assertEquals("hl filenames are the same", file0, file1); } - private String createSnapshot(String vname, String bname) - throws IOException, InterruptedException, TimeoutException { - final OzoneManager om = cluster.getOzoneManager(); - String snapshotName = UUID.randomUUID().toString(); - OzoneManagerProtocol writeClient = cluster.getRpcClient().getObjectStore() - .getClientProxy().getOzoneManagerClient(); - - writeClient.createSnapshot(vname, bname, snapshotName); - SnapshotInfo snapshotInfo = om - .getMetadataManager().getSnapshotInfoTable() - .get(SnapshotInfo.getTableKey(vname, bname, snapshotName)); - String snapshotPath = getSnapshotPath(conf, snapshotInfo) - + OM_KEY_PREFIX; - GenericTestUtils.waitFor(() -> new File(snapshotPath).exists(), - 100, 2000); - return snapshotPath; - } - - private void prepArchiveData() throws Exception { - - setupCluster(); - metaDir = OMStorage.getOmDbDir(conf); - - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), - "content"); - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), - "content"); - - // this sleep can be removed after this is fixed: - // https://issues.apache.org/jira/browse/HDDS-7279 - Thread.sleep(2000); - snapshotDirName = - createSnapshot(bucket.getVolumeName(), bucket.getName()); - snapshotDirName2 = - createSnapshot(bucket.getVolumeName(), bucket.getName()); - - - // create dummy link from one snapshot dir to the other - // to confirm that links are recognized even if - // they are not in the checkpoint directory - Path dummyFile = Paths.get(snapshotDirName, "dummyFile"); - Path dummyLink = Paths.get(snapshotDirName2, "dummyFile"); - - compactionDirPath = Paths.get(metaDir.toString(), - OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_BACKUP_DIR); - Path dummyLink2 = Paths.get(compactionDirPath.toString(), "dummyFile"); - Files.write(dummyFile, "dummyData".getBytes(StandardCharsets.UTF_8)); - Files.createLink(dummyLink, dummyFile); - Files.createLink(dummyLink2, dummyFile); - Path currentFile = Paths.get(metaDir.toString(), - OM_DB_NAME, "CURRENT"); - Path currentLink = Paths.get(compactionDirPath.toString(), "CURRENT"); - Files.createLink(currentLink, currentFile); - - dbCheckpoint = cluster.getOzoneManager() - .getMetadataManager().getStore() - .getCheckpoint(true); - - } - } From 07eac3bd97843789a8f2d20d39ea0fedd58ed101 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 17 Nov 2022 15:06:09 -0800 Subject: [PATCH 054/120] cleanup --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 100 ++++-------------- 1 file changed, 19 insertions(+), 81 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index af21b7507a6..74ee624eb05 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -60,6 +60,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeoutException; import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithData.createKey; import static org.junit.Assert.assertTrue; @@ -161,6 +162,8 @@ public void testInstallSnapshot() throws Exception { // Do some transactions so that the log index increases List keys = writeKeysToIncreaseLogIndex(leaderRatisServer, 200); + createSnapshot(leaderOM, keys); + // Get the latest db checkpoint from the leader OM. TransactionInfo transactionInfo = TransactionInfo.readTransactionInfo(leaderOM.getMetadataManager()); @@ -231,6 +234,20 @@ public void testInstallSnapshot() throws Exception { getDefaultBucketLayout()).get(followerOMMetaMngr.getOzoneKey( volumeName, bucketName, newKeys.get(0)))); */ + + // read back data from the snapshot + OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(".snapshot/snap1/" + keys.get(0)).build(); + OmKeyInfo omKeyInfo = null; + try { + omKeyInfo = followerOM.lookupKey(omKeyArgs); + } catch (Exception e) { + Assertions.fail("received exception: " + e); + } + Assertions.assertTrue(omKeyInfo != null); + Assertions.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); } @Ignore("Enable this unit test after RATIS-1481 used") @@ -541,28 +558,8 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { Assert.assertTrue(logCapture.getOutput().contains(msg)); } - @Test - @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE", - "NP_NULL_ON_SOME_PATH"}) - public void testOmSnapshots() throws Exception { - // Get the leader OM - String leaderOMNodeId = OmFailoverProxyUtil - .getFailoverProxyProvider(objectStore.getClientProxy()) - .getCurrentProxyOMNodeId(); - - OzoneManager leaderOM = cluster.getOzoneManager(leaderOMNodeId); - OzoneManagerRatisServer leaderRatisServer = leaderOM.getOmRatisServer(); - - // Find the inactive OM - String followerNodeId = leaderOM.getPeerNodes().get(0).getNodeId(); - if (cluster.isOMActive(followerNodeId)) { - followerNodeId = leaderOM.getPeerNodes().get(1).getNodeId(); - } - OzoneManager followerOM = cluster.getOzoneManager(followerNodeId); - - // Do some transactions so that the log index increases - List keys = writeKeysToIncreaseLogIndex(leaderRatisServer, 200); - + private void createSnapshot(OzoneManager leaderOM, List keys) + throws TimeoutException, InterruptedException, IOException { // Avoid double buffer issue waiting for keys GenericTestUtils.waitFor(() -> { try { @@ -589,65 +586,6 @@ public void testOmSnapshots() throws Exception { return false; } }, 100, 3000); - // Get the latest db checkpoint from the leader OM. - TransactionInfo transactionInfo = - TransactionInfo.readTransactionInfo(leaderOM.getMetadataManager()); - TermIndex leaderOMTermIndex = - TermIndex.valueOf(transactionInfo.getTerm(), - transactionInfo.getTransactionIndex()); - long leaderOMSnapshotIndex = leaderOMTermIndex.getIndex(); - long leaderOMSnapshotTermIndex = leaderOMTermIndex.getTerm(); - - - - // Start the inactive OM. Checkpoint installation will happen spontaneously. - cluster.startInactiveOM(followerNodeId); - GenericTestUtils.LogCapturer logCapture = - GenericTestUtils.LogCapturer.captureLogs(OzoneManager.LOG); - - // The recently started OM should be lagging behind the leader OM. - // Wait & for follower to update transactions to leader snapshot index. - // Timeout error if follower does not load update within 3s - GenericTestUtils.waitFor(() -> - followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex() - >= leaderOMSnapshotIndex - 1, 100, 3000); - - long followerOMLastAppliedIndex = - followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex(); - assertTrue( - followerOMLastAppliedIndex >= leaderOMSnapshotIndex - 1); - - // After the new checkpoint is installed, the follower OM - // lastAppliedIndex must >= the snapshot index of the checkpoint. It - // could be great than snapshot index if there is any conf entry from ratis. - followerOMLastAppliedIndex = followerOM.getOmRatisServer() - .getLastAppliedTermIndex().getIndex(); - assertTrue(followerOMLastAppliedIndex >= leaderOMSnapshotIndex); - assertTrue(followerOM.getOmRatisServer().getLastAppliedTermIndex() - .getTerm() >= leaderOMSnapshotTermIndex); - - // Verify RPC server is running - GenericTestUtils.waitFor(() -> { - return followerOM.isOmRpcServerRunning(); - }, 100, 5000); - - // Verify checkpoint installation was happened. - String msg = "Reloaded OM state"; - Assert.assertTrue(logCapture.getOutput().contains(msg)); - - // read back data from the snapshot - OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(".snapshot/snap1/" + keys.get(0)).build(); - OmKeyInfo omKeyInfo = null; - try { - omKeyInfo = followerOM.lookupKey(omKeyArgs); - } catch (Exception e) { - Assertions.fail("received exception: " + e); - } - Assertions.assertTrue(omKeyInfo != null); - Assertions.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); } private List writeKeysToIncreaseLogIndex( From c4f81571b720e32cb5f8a4a7a1c775998e3311dc Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 17 Nov 2022 16:04:42 -0800 Subject: [PATCH 055/120] removed pom.xml/differ changes --- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 6 ++++-- hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 4 ---- .../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 15 ++++++--------- .../ozone/om/TestOMDbCheckpointServlet.java | 2 -- .../hadoop/ozone/om/TestOmSnapshotManager.java | 5 ++++- 5 files changed, 14 insertions(+), 18 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 415fdf51640..96dcb0e04f0 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -49,7 +49,9 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_BACKUP_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_LOG_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DIR; /** * RocksDB Store that supports creating Tables in DB. @@ -92,8 +94,8 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, try { if (enableCompactionLog) { rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer( - dbLocation.getParent(), OM_COMPACTION_BACKUP_DIR, - OM_COMPACTION_LOG_DIR, dbLocation); + dbLocation.getParent() + OM_KEY_PREFIX + OM_SNAPSHOT_DIFF_DIR, + OM_COMPACTION_BACKUP_DIR, OM_COMPACTION_LOG_DIR, dbLocation); rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions); } else { rocksDBCheckpointDiffer = null; diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml index 86661c51ac6..0c03876df08 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml +++ b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml @@ -80,10 +80,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.ozone hdds-test-utils - - org.apache.ozone - hdds-common - diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 6d3431c0aed..9f19395d56a 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -35,7 +35,6 @@ import org.rocksdb.TableProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.ozone.OzoneConsts; import java.io.BufferedWriter; import java.io.File; @@ -141,17 +140,16 @@ public class RocksDBCheckpointDiffer { * Constructor. * Note that previous compaction logs are loaded by RDBStore after this * object's initialization by calling loadAllCompactionLogs(). - * @param metadataDir Ozone metadata directory. + * @param snapshotDiffDir parent directory for diff state * @param sstBackupDir Name of the SST backup dir under metadata dir. * @param compactionLogDirName Name of the compaction log dir. */ - public RocksDBCheckpointDiffer(String metadataDir, String sstBackupDir, + public RocksDBCheckpointDiffer(String snapshotDiffDir, String sstBackupDir, String compactionLogDirName, File activeDBLocation) { - setCompactionLogDir(metadataDir, compactionLogDirName); + setCompactionLogDir(snapshotDiffDir, compactionLogDirName); - this.sstBackupDir = Paths.get(metadataDir, - OzoneConsts.OM_SNAPSHOT_DIFF_DIR, sstBackupDir) + "/"; + this.sstBackupDir = Paths.get(snapshotDiffDir, sstBackupDir) + "/"; // Create the directory if SST backup path does not already exist File dir = new File(this.sstBackupDir); @@ -166,11 +164,10 @@ public RocksDBCheckpointDiffer(String metadataDir, String sstBackupDir, this.activeDBLocationStr = activeDBLocation.toString() + "/"; } - private void setCompactionLogDir(String metadataDir, + private void setCompactionLogDir(String snapshotDiffDir, String compactionLogDirName) { - final File parentDir = new File(metadataDir, - OzoneConsts.OM_SNAPSHOT_DIFF_DIR); + final File parentDir = new File(snapshotDiffDir); if (!parentDir.exists()) { if (!parentDir.mkdirs()) { LOG.error("Error creating compaction log parent dir."); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 85610a86389..ad99683e8b2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -338,8 +338,6 @@ public void testWriteDbDataToStream() truncateFileName(metaDirLength, Paths.get(snapshotDirName2)); String shortCompactionDirLocation = truncateFileName(metaDirLength, compactionDirPath); - Path finalSnapshotLocation = - Paths.get(testDirName, shortSnapshotLocation); Set finalFullSet = getFiles(Paths.get(testDirName, OM_SNAPSHOT_DIR), testDirLength); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index f9ecd7cc5c0..ff3c12eeb8b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -127,6 +128,7 @@ public void testCloseOnEviction() throws IOException { } @Test + @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH"}) public void testHardLinkCreation() throws IOException { byte[] dummyData = {0}; @@ -172,7 +174,8 @@ public void testHardLinkCreation() throws IOException { Path value = entry.getValue(); // convert checkpoint path to om.db if (value.toString().contains(OM_CHECKPOINT_DIR)) { - value = Paths.get(dbDir.toString(), value.getFileName().toString()); + value = Paths.get(dbDir.toString(), + value.getFileName().toString()); } Assert.assertEquals("link matches original file", getINode(entry.getKey()), getINode(value)); From fa3e691ddbc54cc4ad69d2d39f86f24b637baeb0 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 17 Nov 2022 16:20:53 -0800 Subject: [PATCH 056/120] fixed intellij warnings --- .../apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java | 3 ++- .../org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +- .../org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 2 +- .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 6 ++---- .../org/apache/hadoop/ozone/om/TestOmSnapshotManager.java | 4 ++-- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index ad99683e8b2..0670f0429ad 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -315,7 +315,7 @@ public void testWriteDbDataToStream() // Move snapshot dir to correct location new File(newDbDirName, OM_SNAPSHOT_DIR) - .renameTo(new File(newDbDir.getParent().toString(), OM_SNAPSHOT_DIR)); + .renameTo(new File(newDbDir.getParent(), OM_SNAPSHOT_DIR)); // Confirm the checkpoint directories match, (after remove extras) @@ -507,6 +507,7 @@ private void checkFabricatedLines(Set directories, List lines, } } + Assert.assertNotNull("real directory found", realDir); directories.remove(realDir); Iterator directoryIterator = directories.iterator(); String dir0 = directoryIterator.next(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 74ee624eb05..95622478a97 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -235,7 +235,7 @@ public void testInstallSnapshot() throws Exception { volumeName, bucketName, newKeys.get(0)))); */ - // read back data from the snapshot + // read back data from the OM snapshot OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) .setBucketName(bucketName) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 9420d3b239c..e065f90aa5a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -206,7 +206,7 @@ private void waitForDirToExist(Path dir) private void processDir(Path dir, Map copyFiles, Map hardLinkFiles) - throws IOException, InterruptedException { + throws IOException { try (Stream files = Files.list(dir)) { for (Path file : files.collect(Collectors.toList())) { if (file.toFile().isDirectory()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index d113c6150fe..579a4086da4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -50,7 +50,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -180,9 +179,8 @@ public static DBCheckpoint createOmSnapshotCheckpoint( @VisibleForTesting static Object getINode(Path file) throws IOException { - Object key = Files.readAttributes( + return Files.readAttributes( file, BasicFileAttributes.class).fileKey(); - return key; } // Get OmSnapshot if the keyname has ".snapshot" key indicator @@ -347,7 +345,7 @@ static void createHardLinks(Path dbPath) throws IOException { } if (!hardLinkFile.delete()) { throw new IOException( - "Failed to delete: " + hardLinkFile.toString()); + "Failed to delete: " + hardLinkFile); } } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index ff3c12eeb8b..97d15b82705 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -137,14 +137,14 @@ public void testHardLinkCreation() throws IOException { File snapDir1 = new File(testDir.toString(), OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); if (!snapDir1.mkdirs()) { - throw new IOException("failed to make directory: " + snapDir1.toString()); + throw new IOException("failed to make directory: " + snapDir1); } Files.write(Paths.get(snapDir1.toString(), "s1"), dummyData); File snapDir2 = new File(testDir.toString(), OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir2"); if (!snapDir2.mkdirs()) { - throw new IOException("failed to make directory: " + snapDir2.toString()); + throw new IOException("failed to make directory: " + snapDir2); } File dbDir = new File(testDir.toString(), OM_DB_NAME); From 58f5103303c23c246bce6ef417583b33664b1df5 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 17 Nov 2022 16:29:24 -0800 Subject: [PATCH 057/120] checkstyle --- .../ozone/om/TestOMDbCheckpointServlet.java | 25 +++++++++++-------- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 1 - .../ozone/om/OMDBCheckpointServlet.java | 2 +- .../hadoop/ozone/om/OmSnapshotManager.java | 2 +- .../apache/hadoop/ozone/om/OzoneManager.java | 1 - .../ozone/om/snapshot/package-info.java | 23 +++++++++++++++++ 6 files changed, 39 insertions(+), 15 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/package-info.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 0670f0429ad..a33aa4d10f8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -360,7 +360,8 @@ public void testWriteDbDataToStream() } Set directories = Sets.newHashSet( - shortSnapshotLocation, shortSnapshotLocation2, shortCompactionDirLocation); + shortSnapshotLocation, shortSnapshotLocation2, + shortCompactionDirLocation); checkFabricatedLines(directories, fabricatedLinkLines, testDirName); Set initialFullSet = @@ -429,13 +430,15 @@ private void prepSnapshotData() throws Exception { Path fabricatedFile = Paths.get(snapshotDirName, "fabricatedFile"); Path fabricatedLink = Paths.get(snapshotDirName2, "fabricatedFile"); - Files.write(fabricatedFile, "fabricatedData".getBytes(StandardCharsets.UTF_8)); + Files.write(fabricatedFile, + "fabricatedData".getBytes(StandardCharsets.UTF_8)); Files.createLink(fabricatedLink, fabricatedFile); // simulate links from the compaction dir compactionDirPath = Paths.get(metaDir.toString(), - OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_BACKUP_DIR); - Path fabricatedLink2 = Paths.get(compactionDirPath.toString(), "fabricatedFile"); + OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_BACKUP_DIR); + Path fabricatedLink2 = Paths.get(compactionDirPath.toString(), + "fabricatedFile"); Files.createLink(fabricatedLink2, fabricatedFile); Path currentFile = Paths.get(metaDir.toString(), OM_DB_NAME, "CURRENT"); @@ -472,8 +475,8 @@ private Set getFiles(Path path, int truncateLength) // Get all files below path, recursively, (skipping fabricated files) @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) - private Set getFiles(Path path, int truncateLength, Set fileSet) - throws IOException { + private Set getFiles(Path path, int truncateLength, + Set fileSet) throws IOException { try (Stream files = Files.list(path)) { for (Path file : files.collect(Collectors.toList())) { if (file.toFile().isDirectory()) { @@ -487,12 +490,12 @@ private Set getFiles(Path path, int truncateLength, Set fileSet) return fileSet; } - // tests to see that fabricated link lines in hardlink file are properly formatted - // "dir1/fabricatedFile dir2/fabricatedFile" + // tests to see that fabricated link lines in hardlink file are + // properly formatted "dir1/fabricatedFile dir2/fabricatedFile" // - // The "fabricated" files/links are ones I've created by hand to fully test the - // code, (as opposed to the "natural" files/links created by the - // create snapshot process.) + // The "fabricated" files/links are ones I've created by hand to + // fully test the code, (as opposed to the "natural" files/links + // created by the create snapshot process.) @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private void checkFabricatedLines(Set directories, List lines, String testDirName) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 95622478a97..9687100a267 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -16,7 +16,6 @@ */ package org.apache.hadoop.ozone.om; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.ExitManager; import org.apache.hadoop.hdds.conf.OzoneConfiguration; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index e065f90aa5a..f183a713a58 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -211,7 +211,7 @@ private void processDir(Path dir, Map copyFiles, for (Path file : files.collect(Collectors.toList())) { if (file.toFile().isDirectory()) { processDir(file, copyFiles, hardLinkFiles); - } else { + } else { processFile(file, copyFiles, hardLinkFiles); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 579a4086da4..5954562b2c4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -289,7 +289,7 @@ private void verifySnapshotInfoForSnapDiff(final SnapshotInfo fromSnapshot, } } /** - * Create file of links to add to tarball + * Create file of links to add to tarball. * Format of entries are either: * dir1/fileTo fileFrom * for files in active db or: diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 27935c6d5a5..fc14319716e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -231,7 +231,6 @@ import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY; import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE; import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.PREPARE_MARKER_KEY; import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_DIR; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/package-info.java new file mode 100644 index 00000000000..d2f3dc6229d --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +/** + * This package contains OM Snapshot related classes. + */ \ No newline at end of file From cbf14af1c1ae309456a8bd22d9beaa1d653da43b Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 17 Nov 2022 17:53:06 -0800 Subject: [PATCH 058/120] comments --- .../src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index fc14319716e..3ed6ec590ab 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3599,6 +3599,8 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, String dbBackupName = OzoneConsts.OM_DB_BACKUP_PREFIX + lastAppliedIndex + "_" + System.currentTimeMillis(); File dbDir = oldDB.getParentFile(); + + // Backup the active fs and snapshot dirs File dbBackupDir = new File(dbDir, dbBackupName); if (!dbBackupDir.mkdirs()) { throw new IOException("Failed to make db backup dir: " + From 474aaf7edbd18fc0496337153586efe9fec71adc Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 18 Nov 2022 14:59:57 -0800 Subject: [PATCH 059/120] trigger new CI check From 9cb92c441488f1d6a6001ce391f8207559bb9521 Mon Sep 17 00:00:00 2001 From: GeorgeJahad Date: Mon, 28 Nov 2022 14:41:45 -0800 Subject: [PATCH 060/120] Update hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java Co-authored-by: Hemant Kumar --- .../org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index a33aa4d10f8..3a30aa4bbed 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -291,8 +291,7 @@ public void testSpnegoEnabled() throws Exception { } @Test - public void testWriteDbDataToStream() - throws Exception { + public void testWriteDbDataToStream() throws Exception { prepSnapshotData(); // set http param to include snapshot data when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)) From 7825c96b5e71fdbd0309d5d396eaf970d48e5f50 Mon Sep 17 00:00:00 2001 From: GeorgeJahad Date: Mon, 28 Nov 2022 14:42:03 -0800 Subject: [PATCH 061/120] Update hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java Co-authored-by: Hemant Kumar --- .../java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 9687100a267..3cf3848d105 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -237,8 +237,8 @@ public void testInstallSnapshot() throws Exception { // read back data from the OM snapshot OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(".snapshot/snap1/" + keys.get(0)).build(); + .setBucketName(bucketName) + .setKeyName(".snapshot/snap1/" + keys.get(0)).build(); OmKeyInfo omKeyInfo = null; try { omKeyInfo = followerOM.lookupKey(omKeyArgs); From 1953d80604ab828f9e6b30700e886af108c95362 Mon Sep 17 00:00:00 2001 From: GeorgeJahad Date: Mon, 28 Nov 2022 14:42:18 -0800 Subject: [PATCH 062/120] Update hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java Co-authored-by: Hemant Kumar --- .../java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 3cf3848d105..ddad37e6278 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -245,7 +245,7 @@ public void testInstallSnapshot() throws Exception { } catch (Exception e) { Assertions.fail("received exception: " + e); } - Assertions.assertTrue(omKeyInfo != null); + Assertions.assertNotNull(omKeyInfo); Assertions.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); } From f5a4a952629a58c844c2db48c4c6f60885885a55 Mon Sep 17 00:00:00 2001 From: GeorgeJahad Date: Mon, 28 Nov 2022 14:52:17 -0800 Subject: [PATCH 063/120] Update hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java Co-authored-by: Hemant Kumar --- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index f183a713a58..395b760643a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -127,16 +127,12 @@ public void writeDbDataToStream(DBCheckpoint checkpoint, includeSnapshotData(request)); try (CompressorOutputStream gzippedOut = new CompressorStreamFactory() - .createCompressorOutputStream(CompressorStreamFactory.GZIP, - destination)) { - - try (TarArchiveOutputStream archiveOutputStream = - new TarArchiveOutputStream(gzippedOut)) { - archiveOutputStream - .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - - writeFilesToArchive(copyFiles, hardLinkFiles, archiveOutputStream); - } + .createCompressorOutputStream(CompressorStreamFactory.GZIP, destination); + TarArchiveOutputStream archiveOutputStream = + new TarArchiveOutputStream(gzippedOut) + ) { + archiveOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + writeFilesToArchive(copyFiles, hardLinkFiles, archiveOutputStream); } catch (CompressorException e) { throw new IOException( "Can't compress the checkpoint: " + From 9aace54b1b885b8b1206aa7ad87af9ebbbe749e2 Mon Sep 17 00:00:00 2001 From: GeorgeJahad Date: Mon, 28 Nov 2022 14:55:31 -0800 Subject: [PATCH 064/120] Update hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java Co-authored-by: Hemant Kumar --- .../java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 395b760643a..d2e96c0def0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -252,8 +252,7 @@ private void writeFilesToArchive(HashMap copyFiles, // checkpoint files go to root of tarball fixedFile = Paths.get(fixedFile).getFileName().toString(); } - includeFile(file.toFile(), fixedFile, - archiveOutputStream); + includeFile(file.toFile(), fixedFile, archiveOutputStream); } // Create list of hard links From b91843b0e6aa09724198dec9abd4d8fa9923bd67 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 28 Nov 2022 16:23:16 -0800 Subject: [PATCH 065/120] cleanup comments --- .../rocksdiff/RocksDBCheckpointDiffer.java | 2 +- .../ozone/om/TestOMDbCheckpointServlet.java | 50 +++++++++---------- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 6 +-- .../ozone/om/OMDBCheckpointServlet.java | 31 ++++++------ .../ozone/om/OmMetadataManagerImpl.java | 6 +-- .../hadoop/ozone/om/OmSnapshotManager.java | 20 ++++---- .../apache/hadoop/ozone/om/OzoneManager.java | 4 +- .../ozone/om/TestOmSnapshotManager.java | 12 ++--- 8 files changed, 64 insertions(+), 67 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 9f19395d56a..0b004757446 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -140,7 +140,7 @@ public class RocksDBCheckpointDiffer { * Constructor. * Note that previous compaction logs are loaded by RDBStore after this * object's initialization by calling loadAllCompactionLogs(). - * @param snapshotDiffDir parent directory for diff state + * @param snapshotDiffDir Parent directory for diff state. * @param sstBackupDir Name of the SST backup dir under metadata dir. * @param compactionLogDirName Name of the compaction log dir. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 3a30aa4bbed..647e4f86e30 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -293,17 +293,17 @@ public void testSpnegoEnabled() throws Exception { @Test public void testWriteDbDataToStream() throws Exception { prepSnapshotData(); - // set http param to include snapshot data + // Set http param to include snapshot data. when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)) .thenReturn("true"); - // get the tarball + // Get the tarball. try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock, fileOutputStream); } - // Untar the file into a temp folder to be examined + // Untar the file into a temp folder to be examined. String testDirName = folder.newFolder().getAbsolutePath(); int testDirLength = testDirName.length() + 1; String newDbDirName = testDirName + OM_KEY_PREFIX + OM_DB_NAME; @@ -312,12 +312,12 @@ public void testWriteDbDataToStream() throws Exception { newDbDir.mkdirs(); FileUtil.unTar(tempFile, newDbDir); - // Move snapshot dir to correct location + // Move snapshot dir to correct location. new File(newDbDirName, OM_SNAPSHOT_DIR) .renameTo(new File(newDbDir.getParent(), OM_SNAPSHOT_DIR)); - // Confirm the checkpoint directories match, (after remove extras) + // Confirm the checkpoint directories match, (after remove extras). Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); Set initialCheckpointSet = getFiles(checkpointLocation, checkpointLocation.toString().length() + 1); @@ -341,7 +341,7 @@ public void testWriteDbDataToStream() throws Exception { Set finalFullSet = getFiles(Paths.get(testDirName, OM_SNAPSHOT_DIR), testDirLength); - // check each line in the hard link file + // Check each line in the hard link file. Stream lines = Files.lines(Paths.get(newDbDirName, OM_HARDLINK_FILE)); @@ -374,22 +374,22 @@ public void testWriteDbDataWithoutOmSnapshot() throws Exception { prepSnapshotData(); - // set http param to exclude snapshot data + // Set http param to exclude snapshot data. when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)) .thenReturn(null); - // get the tarball + // Get the tarball. try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock, fileOutputStream); } - // Untar the file into a temp folder to be examined + // Untar the file into a temp folder to be examined. String testDirName = folder.newFolder().getAbsolutePath(); int testDirLength = testDirName.length() + 1; FileUtil.unTar(tempFile, new File(testDirName)); - // Confirm the checkpoint directories match + // Confirm the checkpoint directories match. Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); Set initialCheckpointSet = getFiles(checkpointLocation, checkpointLocation.toString().length() + 1); @@ -408,14 +408,14 @@ private void prepSnapshotData() throws Exception { OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); - // Create dummy keys for snapshotting + // Create dummy keys for snapshotting. TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), "content"); TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), "content"); - // this sleep can be removed after this is fixed: - // https://issues.apache.org/jira/browse/HDDS-7279 + // This sleep can be removed after this is fixed: + // https://issues.apache.org/jira/browse/HDDS-7279 Thread.sleep(2000); snapshotDirName = createSnapshot(bucket.getVolumeName(), bucket.getName()); @@ -423,9 +423,9 @@ private void prepSnapshotData() throws Exception { createSnapshot(bucket.getVolumeName(), bucket.getName()); - // create fabricated links to snapshot dirs - // to confirm that links are recognized even if - // they are don't point to the checkpoint directory + // Create fabricated links to snapshot dirs + // to confirm that links are recognized even if + // they are don't point to the checkpoint directory. Path fabricatedFile = Paths.get(snapshotDirName, "fabricatedFile"); Path fabricatedLink = Paths.get(snapshotDirName2, "fabricatedFile"); @@ -433,7 +433,7 @@ private void prepSnapshotData() throws Exception { "fabricatedData".getBytes(StandardCharsets.UTF_8)); Files.createLink(fabricatedLink, fabricatedFile); - // simulate links from the compaction dir + // Simulate links from the compaction dir. compactionDirPath = Paths.get(metaDir.toString(), OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_BACKUP_DIR); Path fabricatedLink2 = Paths.get(compactionDirPath.toString(), @@ -472,7 +472,7 @@ private Set getFiles(Path path, int truncateLength) return getFiles(path, truncateLength, new HashSet<>()); } - // Get all files below path, recursively, (skipping fabricated files) + // Get all files below path, recursively, (skipping fabricated files). @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private Set getFiles(Path path, int truncateLength, Set fileSet) throws IOException { @@ -489,12 +489,12 @@ private Set getFiles(Path path, int truncateLength, return fileSet; } - // tests to see that fabricated link lines in hardlink file are - // properly formatted "dir1/fabricatedFile dir2/fabricatedFile" + // Tests to see that fabricated link lines in hardlink file are + // properly formatted "dir1/fabricatedFile dir2/fabricatedFile". // - // The "fabricated" files/links are ones I've created by hand to - // fully test the code, (as opposed to the "natural" files/links - // created by the create snapshot process.) + // The "fabricated" files/links are ones I've created by hand to + // fully test the code, (as opposed to the "natural" files/links + // created by the create snapshot process). @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private void checkFabricatedLines(Set directories, List lines, String testDirName) { @@ -530,8 +530,8 @@ private void checkFabricatedLines(Set directories, List lines, } } - // validates line in hard link file. should look something like: - // "dir1/x.sst x.sst" + // Validates line in hard link file. should look something like: + // "dir1/x.sst x.sst". private void checkLine(String shortSnapshotLocation, String shortSnapshotLocation2, String line) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index ddad37e6278..7a3c397eb26 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -234,7 +234,7 @@ public void testInstallSnapshot() throws Exception { volumeName, bucketName, newKeys.get(0)))); */ - // read back data from the OM snapshot + // Read back data from the OM snapshot. OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) .setBucketName(bucketName) @@ -559,7 +559,7 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { private void createSnapshot(OzoneManager leaderOM, List keys) throws TimeoutException, InterruptedException, IOException { - // Avoid double buffer issue waiting for keys + // Avoid double buffer issue waiting for keys. GenericTestUtils.waitFor(() -> { try { OmKeyInfo key = leaderOM.getMetadataManager() @@ -573,7 +573,7 @@ private void createSnapshot(OzoneManager leaderOM, List keys) }, 100, 10000); objectStore.createSnapshot(volumeName, bucketName, "snap1"); - // allow the snapshot to be written to the info table + // Allow the snapshot to be written to the info table. GenericTestUtils.waitFor(() -> { try { SnapshotInfo snapshotInfo = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index d2e96c0def0..9882f3b48dd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -118,10 +118,10 @@ public void writeDbDataToStream(DBCheckpoint checkpoint, HttpServletRequest request, OutputStream destination) throws IOException, InterruptedException, CompressorException { - // Map of inodes to path - HashMap copyFiles = new HashMap<>(); - // Map of link to path - HashMap hardLinkFiles = new HashMap<>(); + // Map of inodes to path. + Map copyFiles = new HashMap<>(); + // Map of link to path. + Map hardLinkFiles = new HashMap<>(); getFilesForArchive(checkpoint, copyFiles, hardLinkFiles, includeSnapshotData(request)); @@ -146,7 +146,7 @@ private void getFilesForArchive(DBCheckpoint checkpoint, boolean includeSnapshotData) throws IOException, InterruptedException { - // Get the active fs files + // Get the active fs files. Path dir = checkpoint.getCheckpointLocation(); processDir(dir, copyFiles, hardLinkFiles); @@ -154,7 +154,7 @@ private void getFilesForArchive(DBCheckpoint checkpoint, return; } - // Get the snapshot files + // Get the snapshot files. waitForSnapshotDirs(checkpoint); Path snapshotDir = Paths.get(OMStorage.getOmDbDir(getConf()).toString(), OM_SNAPSHOT_DIR); @@ -162,7 +162,7 @@ private void getFilesForArchive(DBCheckpoint checkpoint, } // The snapshotInfo table may contain a snapshot that - // doesn't yet exist on the fs, so wait a few seconds for it + // doesn't yet exist on the fs, so wait a few seconds for it. private void waitForSnapshotDirs(DBCheckpoint checkpoint) throws IOException, InterruptedException { @@ -176,7 +176,7 @@ private void waitForSnapshotDirs(DBCheckpoint checkpoint) iterator = checkpointMetadataManager .getSnapshotInfoTable().iterator()) { - // for each entry, wait for corresponding directory + // For each entry, wait for corresponding directory. while (iterator.hasNext()) { Table.KeyValue entry = iterator.next(); Path path = Paths.get(getSnapshotPath(conf, entry.getValue())); @@ -192,12 +192,9 @@ private void waitForDirToExist(Path dir) while (!dir.toFile().exists()) { Thread.sleep(100); if (System.currentTimeMillis() > endTime) { - break; + throw new IOException("snapshot dir doesn't exist: " + dir); } } - if (System.currentTimeMillis() > endTime) { - throw new IOException("snapshot dir doesn't exist: " + dir); - } } private void processDir(Path dir, Map copyFiles, @@ -216,9 +213,9 @@ private void processDir(Path dir, Map copyFiles, private void processFile(Path file, Map copyFiles, Map hardLinkFiles) throws IOException { - // get the inode + // Get the inode. Object key = OmSnapshotManager.getINode(file); - // If we already have the inode, store as hard link + // If we already have the inode, store as hard link. if (copyFiles.containsKey(key)) { hardLinkFiles.put(file, copyFiles.get(key)); } else { @@ -226,7 +223,7 @@ private void processFile(Path file, Map copyFiles, } } - // returns value of http request parameter + // Returns value of http request parameter. private boolean includeSnapshotData(HttpServletRequest request) { String includeParam = request.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA); @@ -245,7 +242,7 @@ private void writeFilesToArchive(HashMap copyFiles, File metaDirPath = ServerUtils.getOzoneMetaDirPath(getConf()); int truncateLength = metaDirPath.toString().length() + 1; - // Go through each of the files to be copied and add to archive + // Go through each of the files to be copied and add to archive. for (Path file : copyFiles.values()) { String fixedFile = truncateFileName(truncateLength, file); if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { @@ -255,7 +252,7 @@ private void writeFilesToArchive(HashMap copyFiles, includeFile(file.toFile(), fixedFile, archiveOutputStream); } - // Create list of hard links + // Create list of hard links. if (!hardLinkFiles.isEmpty()) { Path hardLinkFile = createHardLinkList(truncateLength, hardLinkFiles); includeFile(hardLinkFile.toFile(), OmSnapshotManager.OM_HARDLINK_FILE, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 73d4ccfb02e..6c1a84b73c8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -304,9 +304,9 @@ protected OmMetadataManagerImpl() { /** * metadata constructor for checkpoints. * - * @param conf - ozone conf - * @param dir - checkpoint parent directory - * @param name - checkpoint directory name + * @param conf - Ozone conf. + * @param dir - Checkpoint parent directory. + * @param name - Checkpoint directory name. * @throws IOException */ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 5954562b2c4..988d750e8e4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -296,9 +296,9 @@ private void verifySnapshotInfoForSnapDiff(final SnapshotInfo fromSnapshot, * dir1/fileTo dir2/fileFrom * for files in another directory, (either another snapshot dir or * sst compaction backup directory) - * @param truncateLength length of initial path to trim in file path - * @param hardLinkFiles a map of link->file paths - * @return Path to the file of links created + * @param truncateLength - Length of initial path to trim in file path. + * @param hardLinkFiles - Map of link->file paths. + * @return Path to the file of links created. */ @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) static Path createHardLinkList(int truncateLength, @@ -308,7 +308,7 @@ static Path createHardLinkList(int truncateLength, StringBuilder sb = new StringBuilder(); for (Map.Entry entry : hardLinkFiles.entrySet()) { String fixedFile = truncateFileName(truncateLength, entry.getValue()); - // If this file is from the active db, strip the path + // If this file is from the active db, strip the path. if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { fixedFile = Paths.get(fixedFile).getFileName().toString(); } @@ -321,7 +321,7 @@ static Path createHardLinkList(int truncateLength, return data; } - // get the filename without the introductory metadata directory + // Get the filename without the introductory metadata directory static String truncateFileName(int truncateLength, Path file) { return file.toString().substring(truncateLength); } @@ -331,11 +331,11 @@ static void createHardLinks(Path dbPath) throws IOException { File hardLinkFile = new File(dbPath.toString(), OM_HARDLINK_FILE); if (hardLinkFile.exists()) { - // Read file + // Read file. List lines = Files.lines(hardLinkFile.toPath()).collect(Collectors.toList()); - // Create a link for each line + // Create a link for each line. for (String l : lines) { String from = l.split("\t")[1]; String to = l.split("\t")[0]; @@ -351,15 +351,15 @@ static void createHardLinks(Path dbPath) throws IOException { } - // Prepend the full path to the hard link entry entry + // Prepend the full path to the hard link entry entry. @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private static Path getFullPath(Path dbPath, String fileName) { File file = new File(fileName); - // If there is no directory then this file belongs in the db + // If there is no directory then this file belongs in the db. if (file.getName().equals(fileName)) { return Paths.get(dbPath.toString(), fileName); } - // else this file belong in a directory parallel to the db + // Else this file belong in a directory parallel to the db. return Paths.get(dbPath.getParent().toString(), fileName); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 3ed6ec590ab..f581215ca83 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3600,7 +3600,7 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, lastAppliedIndex + "_" + System.currentTimeMillis(); File dbDir = oldDB.getParentFile(); - // Backup the active fs and snapshot dirs + // Backup the active fs and snapshot dirs. File dbBackupDir = new File(dbDir, dbBackupName); if (!dbBackupDir.mkdirs()) { throw new IOException("Failed to make db backup dir: " + @@ -3653,7 +3653,7 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, return dbBackupDir; } - // move the new snapshot directory into place and create hard links + // Move the new snapshot directory into place and create hard links. private void moveOmSnapshotData(Path dbPath, Path dbSnapshotsDir) throws IOException { Path incomingSnapshotsDir = Paths.get(dbPath.toString(), diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 97d15b82705..0f040720a20 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -133,7 +133,7 @@ public void testHardLinkCreation() throws IOException { byte[] dummyData = {0}; - // Create dummy files to be linked to + // Create dummy files to be linked to. File snapDir1 = new File(testDir.toString(), OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); if (!snapDir1.mkdirs()) { @@ -150,7 +150,7 @@ public void testHardLinkCreation() throws IOException { File dbDir = new File(testDir.toString(), OM_DB_NAME); Files.write(Paths.get(dbDir.toString(), "f1"), dummyData); - // Create map of links to dummy files + // Create map of links to dummy files. File checkpointDir1 = new File(testDir.toString(), OM_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); Map hardLinkFiles = new HashMap<>(); @@ -159,20 +159,20 @@ public void testHardLinkCreation() throws IOException { hardLinkFiles.put(Paths.get(snapDir2.toString(), "s1"), Paths.get(snapDir1.toString(), "s1")); - // Create link list + // Create link list. Path hardLinkList = OmSnapshotManager.createHardLinkList( testDir.toString().length() + 1, hardLinkFiles); Files.move(hardLinkList, Paths.get(dbDir.toString(), OM_HARDLINK_FILE)); - // Create links from list + // Create links from list. OmSnapshotManager.createHardLinks(dbDir.toPath()); - // Confirm expected links + // Confirm expected links. for (Map.Entry entry : hardLinkFiles.entrySet()) { Assert.assertTrue(entry.getKey().toFile().exists()); Path value = entry.getValue(); - // convert checkpoint path to om.db + // Convert checkpoint path to om.db. if (value.toString().contains(OM_CHECKPOINT_DIR)) { value = Paths.get(dbDir.toString(), value.getFileName().toString()); From 09ab4ea5b5c5b07346a32b58b790d68ef0e46130 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 29 Nov 2022 09:12:37 -0800 Subject: [PATCH 066/120] cleanup --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 6 +----- .../org/apache/hadoop/ozone/om/OzoneManager.java | 15 ++++----------- .../hadoop/ozone/om/TestOmSnapshotManager.java | 1 - 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 7a3c397eb26..95f76fa7c15 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -240,11 +240,7 @@ public void testInstallSnapshot() throws Exception { .setBucketName(bucketName) .setKeyName(".snapshot/snap1/" + keys.get(0)).build(); OmKeyInfo omKeyInfo = null; - try { - omKeyInfo = followerOM.lookupKey(omKeyArgs); - } catch (Exception e) { - Assertions.fail("received exception: " + e); - } + omKeyInfo = followerOM.lookupKey(omKeyArgs); Assertions.assertNotNull(omKeyInfo); Assertions.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index f581215ca83..ac64dd34253 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3609,16 +3609,10 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, File dbBackup = new File(dbBackupDir, oldDB.getName()); File dbSnapshotsDir = new File(dbDir, OM_SNAPSHOT_DIR); File dbSnapshotsBackup = new File(dbBackupDir, OM_SNAPSHOT_DIR); - try { - Files.move(oldDB.toPath(), dbBackup.toPath()); - if (dbSnapshotsDir.exists()) { - Files.move(dbSnapshotsDir.toPath(), - dbSnapshotsBackup.toPath()); - } - } catch (IOException e) { - LOG.error("Failed to create a backup of the current DB. Aborting " + - "snapshot installation."); - throw e; + Files.move(oldDB.toPath(), dbBackup.toPath()); + if (dbSnapshotsDir.exists()) { + Files.move(dbSnapshotsDir.toPath(), + dbSnapshotsBackup.toPath()); } // Move the new DB checkpoint into the om metadata dir @@ -3662,7 +3656,6 @@ private void moveOmSnapshotData(Path dbPath, Path dbSnapshotsDir) Files.move(incomingSnapshotsDir, dbSnapshotsDir); OmSnapshotManager.createHardLinks(dbPath); } - } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 0f040720a20..784c9e4588f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -132,7 +132,6 @@ public void testCloseOnEviction() throws IOException { public void testHardLinkCreation() throws IOException { byte[] dummyData = {0}; - // Create dummy files to be linked to. File snapDir1 = new File(testDir.toString(), OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "dir1"); From 16c3cf45d4dd203586d230067b1cd93178493390 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 30 Nov 2022 16:37:51 -0800 Subject: [PATCH 067/120] cleanup --- .../rocksdiff/RocksDBCheckpointDiffer.java | 2 +- .../ozone/om/TestOMDbCheckpointServlet.java | 20 ++++++++++++------- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 10 ++++++---- .../ozone/om/OMDBCheckpointServlet.java | 4 ++-- .../ozone/om/OmMetadataManagerImpl.java | 2 +- .../hadoop/ozone/om/OmSnapshotManager.java | 14 +++++++++++-- .../apache/hadoop/ozone/om/OzoneManager.java | 12 ++++++++++- 7 files changed, 46 insertions(+), 18 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 0b004757446..592423dd3ba 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -176,7 +176,7 @@ private void setCompactionLogDir(String snapshotDiffDir, } this.compactionLogDir = - Paths.get(parentDir.toString(), compactionLogDirName).toString(); + Paths.get(parentDir.getPath(), compactionLogDirName).toString(); File clDir = new File(compactionLogDir); if (!clDir.exists() && !clDir.mkdir()) { LOG.error("Error creating compaction log dir."); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 647e4f86e30..b6e34db0df6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -489,15 +489,21 @@ private Set getFiles(Path path, int truncateLength, return fileSet; } - // Tests to see that fabricated link lines in hardlink file are - // properly formatted "dir1/fabricatedFile dir2/fabricatedFile". - // - // The "fabricated" files/links are ones I've created by hand to - // fully test the code, (as opposed to the "natural" files/links - // created by the create snapshot process). + /** + * Confirm fabricated link lines in hardlink file are properly + * formatted: "dir1/fabricatedFile dir2/fabricatedFile". + * + * The "fabricated" files/links are ones I've created by hand to + * fully test the code, (as opposed to the "natural" files/links + * created by the create snapshot process). + * + * @param directories Possible directories for the links to exist in. + * @param lines Text lines defining the link paths. + * @param testDirName Name of test directory. + */ @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private void checkFabricatedLines(Set directories, List lines, - String testDirName) { + String testDirName) { // find the real file String realDir = null; for (String dir: directories) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 95f76fa7c15..8e120a5bd16 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -572,10 +572,12 @@ private void createSnapshot(OzoneManager leaderOM, List keys) // Allow the snapshot to be written to the info table. GenericTestUtils.waitFor(() -> { try { - SnapshotInfo snapshotInfo = - leaderOM.getMetadataManager().getSnapshotInfoTable() - .getSkipCache( - SnapshotInfo.getTableKey(volumeName, bucketName, "snap1")); + String tableKey = SnapshotInfo.getTableKey(volumeName, + bucketName, + "snap1"); + SnapshotInfo snapshotInfo = leaderOM.getMetadataManager() + .getSnapshotInfoTable() + .getSkipCache(tableKey); return snapshotInfo != null; } catch (Exception e) { return false; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 9882f3b48dd..792cc0f2485 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -234,8 +234,8 @@ private boolean includeSnapshotData(HttpServletRequest request) { } @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) - private void writeFilesToArchive(HashMap copyFiles, - HashMap hardLinkFiles, + private void writeFilesToArchive(Map copyFiles, + Map hardLinkFiles, ArchiveOutputStream archiveOutputStream) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 6c1a84b73c8..2e011bad1c4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -302,7 +302,7 @@ protected OmMetadataManagerImpl() { } /** - * metadata constructor for checkpoints. + * Metadata constructor for checkpoints. * * @param conf - Ozone conf. * @param dir - Checkpoint parent directory. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 988d750e8e4..0b1976102f5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -321,12 +321,22 @@ static Path createHardLinkList(int truncateLength, return data; } - // Get the filename without the introductory metadata directory + /** + * Get the filename without the introductory metadata directory + * + * @param truncateLength Length to remove. + * @param file File to remove prefix from. + * @return Truncated string. + */ static String truncateFileName(int truncateLength, Path file) { return file.toString().substring(truncateLength); } - // Create hard links listed in OM_HARDLINK_FILE + /** + * Create hard links listed in OM_HARDLINK_FILE + * + * @param dbPath Path to db to have links created. + */ static void createHardLinks(Path dbPath) throws IOException { File hardLinkFile = new File(dbPath.toString(), OM_HARDLINK_FILE); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index ac64dd34253..3f1c97ab3c1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3383,6 +3383,9 @@ public TermIndex installSnapshotFromLeader(String leaderId) { } DBCheckpoint omDBCheckpoint = getDBCheckpointFromLeader(leaderId); + if (omDBCheckpoint == null) { + return null; + } LOG.info("Downloaded checkpoint from Leader {} to the location {}", leaderId, omDBCheckpoint.getCheckpointLocation()); @@ -3615,6 +3618,14 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, dbSnapshotsBackup.toPath()); } + moveCheckpointFiles(oldDB, checkpointPath, dbDir, dbBackup, dbSnapshotsDir, + dbSnapshotsBackup); + return dbBackupDir; + } + + private void moveCheckpointFiles(File oldDB, Path checkpointPath, File dbDir, + File dbBackup, File dbSnapshotsDir, + File dbSnapshotsBackup) throws IOException { // Move the new DB checkpoint into the om metadata dir Path markerFile = new File(dbDir, DB_TRANSIENT_MARKER).toPath(); try { @@ -3644,7 +3655,6 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, } throw e; } - return dbBackupDir; } // Move the new snapshot directory into place and create hard links. From e3a29217e37c7160c2bb3805e1b831fbc4c617d6 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 30 Nov 2022 17:49:10 -0800 Subject: [PATCH 068/120] Skips unexpected snapshot directories --- .../ozone/om/TestOMDbCheckpointServlet.java | 9 ++++- .../ozone/om/OMDBCheckpointServlet.java | 33 ++++++++++++++----- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index b6e34db0df6..0c1be89aab0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -422,6 +422,13 @@ private void prepSnapshotData() throws Exception { snapshotDirName2 = createSnapshot(bucket.getVolumeName(), bucket.getName()); + // Create dummy snapshot to make sure it is not included. + Path fabricatedSnapshot = Paths.get( + new File(snapshotDirName).getParent(), + "fabricatedSnapshot"); + fabricatedSnapshot.toFile().mkdirs(); + Paths.get(fabricatedSnapshot.toString(), "fabricatedFile") + .toFile().createNewFile(); // Create fabricated links to snapshot dirs // to confirm that links are recognized even if @@ -481,7 +488,7 @@ private Set getFiles(Path path, int truncateLength, if (file.toFile().isDirectory()) { getFiles(file, truncateLength, fileSet); } - if (!file.getFileName().toString().equals("fabricatedFile")) { + if (!file.getFileName().toString().startsWith("fabricated")) { fileSet.add(truncateFileName(truncateLength, file)); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 792cc0f2485..626a356fa36 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -49,12 +49,15 @@ import java.time.Duration; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; import static org.apache.hadoop.ozone.om.OmSnapshotManager.createHardLinkList; @@ -148,26 +151,32 @@ private void getFilesForArchive(DBCheckpoint checkpoint, // Get the active fs files. Path dir = checkpoint.getCheckpointLocation(); - processDir(dir, copyFiles, hardLinkFiles); + processDir(dir, copyFiles, hardLinkFiles, new HashSet<>()); if (!includeSnapshotData) { return; } // Get the snapshot files. - waitForSnapshotDirs(checkpoint); + Set snapshotPaths = waitForSnapshotDirs(checkpoint); Path snapshotDir = Paths.get(OMStorage.getOmDbDir(getConf()).toString(), OM_SNAPSHOT_DIR); - processDir(snapshotDir, copyFiles, hardLinkFiles); + processDir(snapshotDir, copyFiles, hardLinkFiles, snapshotPaths); } - // The snapshotInfo table may contain a snapshot that - // doesn't yet exist on the fs, so wait a few seconds for it. - private void waitForSnapshotDirs(DBCheckpoint checkpoint) + /** + * The snapshotInfo table may contain a snapshot that + * doesn't yet exist on the fs, so wait a few seconds for it. + * @param checkpoint Checkpoint containing snapshot entries expected. + * @return Set of expected snapshot dirs. + */ + private Set waitForSnapshotDirs(DBCheckpoint checkpoint) throws IOException, InterruptedException { OzoneConfiguration conf = getConf(); + Set snapshotPaths = new HashSet<>(); + // get snapshotInfo entries OmMetadataManagerImpl checkpointMetadataManager = OmMetadataManagerImpl.createCheckpointMetadataManager( @@ -181,8 +190,10 @@ private void waitForSnapshotDirs(DBCheckpoint checkpoint) Table.KeyValue entry = iterator.next(); Path path = Paths.get(getSnapshotPath(conf, entry.getValue())); waitForDirToExist(path); + snapshotPaths.add(path); } } + return snapshotPaths; } private void waitForDirToExist(Path dir) @@ -198,12 +209,18 @@ private void waitForDirToExist(Path dir) } private void processDir(Path dir, Map copyFiles, - Map hardLinkFiles) + Map hardLinkFiles, + Set snapshotPaths) throws IOException { try (Stream files = Files.list(dir)) { for (Path file : files.collect(Collectors.toList())) { if (file.toFile().isDirectory()) { - processDir(file, copyFiles, hardLinkFiles); + // Skip any unexpected snapshot files. + if (file.getParent().toString().endsWith(OM_SNAPSHOT_CHECKPOINT_DIR) + && !snapshotPaths.contains(file)) { + continue; + } + processDir(file, copyFiles, hardLinkFiles, snapshotPaths); } else { processFile(file, copyFiles, hardLinkFiles); } From d711d9ae6b7c9fcc0c5c5129aa005a22b8479340 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 1 Dec 2022 13:58:50 -0800 Subject: [PATCH 069/120] checkstyle --- .../org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 7 +++++-- .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 626a356fa36..bec2034f430 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -130,11 +130,13 @@ public void writeDbDataToStream(DBCheckpoint checkpoint, includeSnapshotData(request)); try (CompressorOutputStream gzippedOut = new CompressorStreamFactory() - .createCompressorOutputStream(CompressorStreamFactory.GZIP, destination); + .createCompressorOutputStream( + CompressorStreamFactory.GZIP, destination); TarArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(gzippedOut) ) { - archiveOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + archiveOutputStream + .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); writeFilesToArchive(copyFiles, hardLinkFiles, archiveOutputStream); } catch (CompressorException e) { throw new IOException( @@ -208,6 +210,7 @@ private void waitForDirToExist(Path dir) } } + @NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE private void processDir(Path dir, Map copyFiles, Map hardLinkFiles, Set snapshotPaths) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 0b1976102f5..bb898af820f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -322,7 +322,7 @@ static Path createHardLinkList(int truncateLength, } /** - * Get the filename without the introductory metadata directory + * Get the filename without the introductory metadata directory. * * @param truncateLength Length to remove. * @param file File to remove prefix from. @@ -333,7 +333,7 @@ static String truncateFileName(int truncateLength, Path file) { } /** - * Create hard links listed in OM_HARDLINK_FILE + * Create hard links listed in OM_HARDLINK_FILE. * * @param dbPath Path to db to have links created. */ From 5004374900cffcea97a2144af1642c54e954b7da Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 1 Dec 2022 14:01:28 -0800 Subject: [PATCH 070/120] findbugs --- .../java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index bec2034f430..ac3d0006355 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -210,7 +210,7 @@ private void waitForDirToExist(Path dir) } } - @NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE + @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE") private void processDir(Path dir, Map copyFiles, Map hardLinkFiles, Set snapshotPaths) From ab05184751fbc08126b9f0751db6247498360a33 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 26 Jan 2023 13:09:55 -0800 Subject: [PATCH 071/120] fixed for updated directory structure --- .../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 11 ++++------- .../apache/hadoop/ozone/om/SstFilteringService.java | 5 ++++- .../hadoop/ozone/om/TestSstFilteringService.java | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java index 4b7ff2e1870..4e0a6ef6536 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java @@ -63,9 +63,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.CONTAINS_SNAPSHOT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; @@ -88,7 +86,7 @@ public class TestOmSnapshot { private static BucketLayout bucketLayout = BucketLayout.LEGACY; private static boolean enabledFileSystemPaths; private static ObjectStore store; - private static File metaDir; + private static OzoneConfiguration leaderConfig; private static OzoneManager leaderOzoneManager; private static OzoneBucket ozoneBucket; @@ -151,7 +149,7 @@ private void init() throws Exception { bucketName = ozoneBucket.getName(); leaderOzoneManager = ((MiniOzoneHAClusterImpl) cluster).getOMLeader(); - OzoneConfiguration leaderConfig = leaderOzoneManager.getConfiguration(); + leaderConfig = leaderOzoneManager.getConfiguration(); cluster.setConf(leaderConfig); OzoneClient client = cluster.getClient(); @@ -163,7 +161,6 @@ private void init() throws Exception { // stop the deletion services so that keys can still be read keyManager.stop(); - metaDir = OMStorage.getOmDbDir(leaderConfig); } @AfterClass @@ -595,8 +592,8 @@ private String createSnapshot(String volName, String buckName, leaderOzoneManager.getMetadataManager().getSnapshotInfoTable() .get(SnapshotInfo.getTableKey(volName, buckName, snapshotName)); String snapshotDirName = - metaDir + OM_KEY_PREFIX + OM_SNAPSHOT_DIR + OM_KEY_PREFIX + OM_DB_NAME - + snapshotInfo.getCheckpointDirName() + OM_KEY_PREFIX + "CURRENT"; + OmSnapshotManager.getSnapshotPath(leaderConfig, snapshotInfo) + + OM_KEY_PREFIX + "CURRENT"; GenericTestUtils .waitFor(() -> new File(snapshotDirName).exists(), 1000, 120000); return snapshotKeyPrefix; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index 92bd08780f0..7696c37f21d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -51,6 +51,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.FILTERED_SNAPSHOTS; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_SST_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_SST_DELETING_LIMIT_PER_TASK_DEFAULT; @@ -143,8 +144,10 @@ public BackgroundTaskResult call() throws Exception { String dbName = OM_DB_NAME + snapshotInfo.getCheckpointDirName(); + String snapshotCheckpointDir = omMetadataDir + OM_KEY_PREFIX + + OM_SNAPSHOT_CHECKPOINT_DIR; RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl - .loadDB(ozoneManager.getConfiguration(), new File(snapshotDir), + .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), dbName, true); RocksDatabase db = rdbStore.getDb(); db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java index acc2803c500..c4b8db4a2de 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java @@ -205,8 +205,7 @@ public void testIrrelevantSstFileDeletion() String dbSnapshots = rocksDbDir + OM_KEY_PREFIX + OM_SNAPSHOT_DIR; String snapshotDirName = - dbSnapshots + OM_KEY_PREFIX + OM_DB_NAME + snapshotInfo - .getCheckpointDirName(); + OmSnapshotManager.getSnapshotPath(conf, snapshotInfo); for (LiveFileMetaData file : allFiles) { File sstFile = From 0b0abbd5f6ff55c94e3acd9156f06b143eca348f Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 6 Feb 2023 15:09:52 -0800 Subject: [PATCH 072/120] merge cleanup --- .../hdds/utils/DBCheckpointServlet.java | 7 ++ .../hadoop/hdds/utils/HddsServerUtil.java | 2 +- .../ozone/om/TestOMDbCheckpointServlet.java | 97 ++++++++++++++----- .../ozone/om/OMDBCheckpointServlet.java | 15 +-- 4 files changed, 86 insertions(+), 35 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 107c67eb02f..35f3c8ad3be 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -22,6 +22,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; @@ -187,5 +188,11 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { } } } + public void writeDbDataToStream(DBCheckpoint checkpoint, + HttpServletRequest ignoredRequest, + OutputStream destination) + throws IOException, InterruptedException{ + writeDBCheckpointToStream(checkpoint, destination); + } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java index ca119192849..b8b66abf963 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java @@ -548,7 +548,7 @@ public static void writeDBCheckpointToStream(DBCheckpoint checkpoint, } } - private static void includeFile(File file, String entryName, + public static void includeFile(File file, String entryName, ArchiveOutputStream archiveOutputStream) throws IOException { ArchiveEntry archiveEntry = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 62df4fdcff0..863885e7697 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -60,7 +60,6 @@ import org.apache.commons.io.FileUtils; import static org.apache.hadoop.hdds.recon.ReconConfig.ConfigStrings.OZONE_RECON_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.hdds.utils.HddsServerUtil.writeDBCheckpointToStream; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; @@ -292,30 +291,82 @@ public void testSpnegoEnabled() throws Exception { } @Test - public void testWriteCheckpointToOutputStream() throws Exception { + public void testWriteDbDataToStream() throws Exception { + prepSnapshotData(); + // Set http param to include snapshot data. + when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)) + .thenReturn("true"); + // Get the tarball. + try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { + omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock, + fileOutputStream); + } + + // Untar the file into a temp folder to be examined. String testDirName = folder.newFolder().getAbsolutePath(); - File checkpoint = new File(testDirName, "checkpoint"); - checkpoint.mkdir(); - File file = new File(checkpoint, "temp1.txt"); - OutputStreamWriter writer = new OutputStreamWriter( - new FileOutputStream(file), StandardCharsets.UTF_8); - writer.write("Test data 1"); - writer.close(); - - file = new File(checkpoint, "/temp2.txt"); - writer = new OutputStreamWriter( - new FileOutputStream(file), StandardCharsets.UTF_8); - writer.write("Test data 2"); - writer.close(); - - File outputFile = - new File(Paths.get(testDirName, "output_file.tar").toString()); - TestDBCheckpoint dbCheckpoint = new TestDBCheckpoint( - checkpoint.toPath()); - writeDBCheckpointToStream(dbCheckpoint, - new FileOutputStream(outputFile)); - assertNotNull(outputFile); + int testDirLength = testDirName.length() + 1; + String newDbDirName = testDirName + OM_KEY_PREFIX + OM_DB_NAME; + int newDbDirLength = newDbDirName.length() + 1; + File newDbDir = new File(newDbDirName); + newDbDir.mkdirs(); + FileUtil.unTar(tempFile, newDbDir); + + // Move snapshot dir to correct location. + new File(newDbDirName, OM_SNAPSHOT_DIR) + .renameTo(new File(newDbDir.getParent(), OM_SNAPSHOT_DIR)); + + + // Confirm the checkpoint directories match, (after remove extras). + Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); + Set initialCheckpointSet = getFiles(checkpointLocation, + checkpointLocation.toString().length() + 1); + Path finalCheckpointLocation = Paths.get(newDbDirName); + Set finalCheckpointSet = getFiles(finalCheckpointLocation, + newDbDirLength); + + Assert.assertTrue("hardlink file exists in checkpoint dir", + finalCheckpointSet.contains(OM_HARDLINK_FILE)); + finalCheckpointSet.remove(OM_HARDLINK_FILE); + Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); + + int metaDirLength = metaDir.toString().length() + 1; + String shortSnapshotLocation = + truncateFileName(metaDirLength, Paths.get(snapshotDirName)); + String shortSnapshotLocation2 = + truncateFileName(metaDirLength, Paths.get(snapshotDirName2)); + String shortCompactionDirLocation = + truncateFileName(metaDirLength, compactionDirPath); + + Set finalFullSet = + getFiles(Paths.get(testDirName, OM_SNAPSHOT_DIR), testDirLength); + + // Check each line in the hard link file. + Stream lines = Files.lines(Paths.get(newDbDirName, + OM_HARDLINK_FILE)); + + List fabricatedLinkLines = new ArrayList<>(); + for (String line: lines.collect(Collectors.toList())) { + Assert.assertFalse("CURRENT file is not a hard link", + line.contains("CURRENT")); + if (line.contains("fabricatedFile")) { + fabricatedLinkLines.add(line); + } else { + checkLine(shortSnapshotLocation, shortSnapshotLocation2, line); + // add links to the final set + finalFullSet.add(line.split("\t")[0]); + } + } + + Set directories = Sets.newHashSet( + shortSnapshotLocation, shortSnapshotLocation2, + shortCompactionDirLocation); + checkFabricatedLines(directories, fabricatedLinkLines, testDirName); + + Set initialFullSet = + getFiles(Paths.get(metaDir.toString(), OM_SNAPSHOT_DIR), metaDirLength); + Assert.assertEquals("found expected snapshot files", + initialFullSet, finalFullSet); } @Test diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index ac3d0006355..3d59eb9384d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -56,6 +56,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.hadoop.hdds.utils.HddsServerUtil.includeFile; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; @@ -120,7 +121,7 @@ public void init() throws ServletException { public void writeDbDataToStream(DBCheckpoint checkpoint, HttpServletRequest request, OutputStream destination) - throws IOException, InterruptedException, CompressorException { + throws IOException, InterruptedException { // Map of inodes to path. Map copyFiles = new HashMap<>(); // Map of link to path. @@ -129,19 +130,11 @@ public void writeDbDataToStream(DBCheckpoint checkpoint, getFilesForArchive(checkpoint, copyFiles, hardLinkFiles, includeSnapshotData(request)); - try (CompressorOutputStream gzippedOut = new CompressorStreamFactory() - .createCompressorOutputStream( - CompressorStreamFactory.GZIP, destination); - TarArchiveOutputStream archiveOutputStream = - new TarArchiveOutputStream(gzippedOut) - ) { + try (TarArchiveOutputStream archiveOutputStream = + new TarArchiveOutputStream(destination)) { archiveOutputStream .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); writeFilesToArchive(copyFiles, hardLinkFiles, archiveOutputStream); - } catch (CompressorException e) { - throw new IOException( - "Can't compress the checkpoint: " + - checkpoint.getCheckpointLocation(), e); } } From a900c65bbf7978baca8df771e0687f621c2f04ac Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 2 Feb 2023 08:50:09 -0800 Subject: [PATCH 073/120] bootstrapStateLock --- .../hadoop/ozone/lock/BootstrapStateLock.java | 6 ++++++ .../apache/hadoop/ozone/om/KeyManager.java | 5 +++-- .../hadoop/ozone/om/KeyManagerImpl.java | 8 ++++---- .../apache/hadoop/ozone/om/OzoneManager.java | 1 + .../ozone/om/service/KeyDeletingService.java | 20 ++++++++++++++++++- 5 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java new file mode 100644 index 00000000000..b5fca5eb4ff --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java @@ -0,0 +1,6 @@ +package org.apache.hadoop.ozone.lock; + +public interface BootstrapStateLock { + public void lockBootstrapState(); + public void unlockBootstrapState(); +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index c14e97ea2b5..e60f0882a73 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.om.fs.OzoneManagerFS; import org.apache.hadoop.hdds.utils.BackgroundService; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket; import java.io.IOException; @@ -154,7 +155,7 @@ List getExpiredOpenKeys(Duration expireThreshold, int count, * Returns the instance of Deleting Service. * @return Background service. */ - BackgroundService getDeletingService(); + KeyDeletingService getDeletingService(); OmMultipartUploadList listMultipartUploads(String volumeName, @@ -247,5 +248,5 @@ List getPendingDeletionSubFiles(long volumeId, * Returns the instance of Snapshot SST Filtering service. * @return Background service. */ - BackgroundService getSnapshotSstFilteringService(); + SstFilteringService getSnapshotSstFilteringService(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 82e1f253290..2abb5dd207b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -157,9 +157,9 @@ public class KeyManagerImpl implements KeyManager { private final OzoneBlockTokenSecretManager secretManager; private final boolean grpcBlockTokenEnabled; - private BackgroundService keyDeletingService; + private KeyDeletingService keyDeletingService; - private BackgroundService snapshotSstFilteringService; + private SstFilteringService snapshotSstFilteringService; private final KeyProviderCryptoExtension kmsProvider; private final boolean enableFileSystemPaths; @@ -605,7 +605,7 @@ public OMMetadataManager getMetadataManager() { } @Override - public BackgroundService getDeletingService() { + public KeyDeletingService getDeletingService() { return keyDeletingService; } @@ -618,7 +618,7 @@ public BackgroundService getOpenKeyCleanupService() { return openKeyCleanupService; } - public BackgroundService getSnapshotSstFilteringService() { + public SstFilteringService getSnapshotSstFilteringService() { return snapshotSstFilteringService; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index ac691d59e68..e729a778a65 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3401,6 +3401,7 @@ public TermIndex installSnapshotFromLeader(String leaderId) { if (omRatisSnapshotProvider == null) { LOG.error("OM Snapshot Provider is not configured as there are no peer " + "nodes."); + keyManager.getDeletingService().lockBootstrapState(); return null; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index b0c6b7a13ad..d67b3b481c3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -30,6 +31,7 @@ import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.common.DeleteBlockGroupResult; +import org.apache.hadoop.ozone.lock.BootstrapStateLock; import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; @@ -67,7 +69,8 @@ * metadata accordingly, if scm returns success for keys, then clean up those * keys. */ -public class KeyDeletingService extends BackgroundService { +public class KeyDeletingService extends BackgroundService implements + BootstrapStateLock { private static final Logger LOG = LoggerFactory.getLogger(KeyDeletingService.class); @@ -83,6 +86,7 @@ public class KeyDeletingService extends BackgroundService { private final int keyLimitPerTask; private final AtomicLong deletedKeyCount; private final AtomicLong runCount; + private final Semaphore bootstrapStateLock = new Semaphore(1); public KeyDeletingService(OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient, @@ -141,6 +145,20 @@ private boolean isRatisEnabled() { return ozoneManager.isRatisEnabled(); } + @Override + public void lockBootstrapState() { + try { + bootstrapStateLock.acquire(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + @Override + public void unlockBootstrapState() { + bootstrapStateLock.release(); + } + /** * A key deleting task scans OM DB and looking for a certain number of * pending-deletion keys, sends these keys along with their associated blocks From 9e9700d02c9d691f4dd71d80748a6cadcc17cede Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 6 Feb 2023 17:13:10 -0800 Subject: [PATCH 074/120] fixed snapshot dir in tests --- .../ozone/om/snapshot/TestOzoneSnapshotRestore.java | 10 +++------- .../snapshot/TestOMSnapshotDeleteResponse.java | 12 ++++++------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java index 55db121ab2e..bebd7a90380 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java @@ -57,9 +57,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -71,7 +69,6 @@ public class TestOzoneSnapshotRestore { private static final String OM_SERVICE_ID = "om-service-test-1"; private MiniOzoneCluster cluster; private ObjectStore store; - private File metaDir; private OzoneManager leaderOzoneManager; private OzoneConfiguration clientConf; @@ -120,7 +117,7 @@ public void init() throws Exception { // stop the deletion services so that keys can still be read keyManager.stop(); - metaDir = OMStorage.getOmDbDir(leaderConfig); + OMStorage.getOmDbDir(leaderConfig); } @@ -157,9 +154,8 @@ private String createSnapshot(String volName, String buckName) .getMetadataManager() .getSnapshotInfoTable() .get(SnapshotInfo.getTableKey(volName, buckName, snapshotName)); - String snapshotDirName = metaDir + OM_KEY_PREFIX + - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + OM_DB_NAME + - snapshotInfo.getCheckpointDirName() + OM_KEY_PREFIX + "CURRENT"; + String snapshotDirName = OmSnapshotManager + .getSnapshotPath(clientConf, snapshotInfo) + OM_KEY_PREFIX + "CURRENT"; GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), 1000, 120000); return snapshotKeyPrefix; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java index 67a2afeea79..1853c18be04 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateSnapshotResponse; @@ -54,12 +55,12 @@ public class TestOMSnapshotDeleteResponse { private OMMetadataManager omMetadataManager; private BatchOperation batchOperation; - private String fsPath; + private OzoneConfiguration ozoneConfiguration; @Before public void setup() throws Exception { - OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); - fsPath = folder.newFolder().getAbsolutePath(); + ozoneConfiguration = new OzoneConfiguration(); + String fsPath = folder.newFolder().getAbsolutePath(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration); @@ -102,9 +103,8 @@ public void testAddToDBBatch() throws Exception { omMetadataManager.getStore().commitBatchOperation(batchOperation); // Confirm snapshot directory was created - String snapshotDir = fsPath + OM_KEY_PREFIX + - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + OM_DB_NAME + - snapshotInfo.getCheckpointDirName(); + String snapshotDir = OmSnapshotManager.getSnapshotPath(ozoneConfiguration, + snapshotInfo); Assert.assertTrue((new File(snapshotDir)).exists()); // Confirm table has 1 entry From b4f1afdf461883e9463942774845a4ad62ed90e9 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 7 Feb 2023 11:01:25 -0800 Subject: [PATCH 075/120] cleanup bootstrapLock --- .../org/apache/hadoop/ozone/lock/BootstrapStateLock.java | 2 +- .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 1 - .../hadoop/ozone/om/service/KeyDeletingService.java | 8 ++------ 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java index b5fca5eb4ff..edf0539e7f6 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java @@ -1,6 +1,6 @@ package org.apache.hadoop.ozone.lock; public interface BootstrapStateLock { - public void lockBootstrapState(); + public void lockBootstrapState() throws InterruptedException; public void unlockBootstrapState(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index e729a778a65..ac691d59e68 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3401,7 +3401,6 @@ public TermIndex installSnapshotFromLeader(String leaderId) { if (omRatisSnapshotProvider == null) { LOG.error("OM Snapshot Provider is not configured as there are no peer " + "nodes."); - keyManager.getDeletingService().lockBootstrapState(); return null; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index d67b3b481c3..a769ff711d2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -146,12 +146,8 @@ private boolean isRatisEnabled() { } @Override - public void lockBootstrapState() { - try { - bootstrapStateLock.acquire(); - } catch (InterruptedException e) { - e.printStackTrace(); - } + public void lockBootstrapState() throws InterruptedException { + bootstrapStateLock.acquire(); } @Override From 61da276cc996874d1ada9b874e3ea174fa4103cd Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 7 Feb 2023 14:56:31 -0800 Subject: [PATCH 076/120] changed to BootstrapStateHandler --- ...teLock.java => BootstrapStateHandler.java} | 2 +- .../rocksdiff/RocksDBCheckpointDiffer.java | 26 +++++++++++++++++-- .../ozone/om/OMDBCheckpointServlet.java | 23 +++++++++++++++- .../hadoop/ozone/om/SstFilteringService.java | 22 +++++++++++++++- .../ozone/om/service/KeyDeletingService.java | 4 +-- 5 files changed, 70 insertions(+), 7 deletions(-) rename hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/{BootstrapStateLock.java => BootstrapStateHandler.java} (77%) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java similarity index 77% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java index edf0539e7f6..c5c6751589c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateLock.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java @@ -1,6 +1,6 @@ package org.apache.hadoop.ozone.lock; -public interface BootstrapStateLock { +public interface BootstrapStateHandler { public void lockBootstrapState() throws InterruptedException; public void unlockBootstrapState(); } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index a8eb7904598..f4f30ec12ef 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -24,9 +24,11 @@ import java.util.Collections; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.rocksdb.AbstractEventListener; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; @@ -87,7 +89,8 @@ * It is important to note that compaction log is per-DB instance. Since * each OM DB instance might trigger compactions at different timings. */ -public class RocksDBCheckpointDiffer implements AutoCloseable { +public class RocksDBCheckpointDiffer implements AutoCloseable, + BootstrapStateHandler { private static final Logger LOG = LoggerFactory.getLogger(RocksDBCheckpointDiffer.class); @@ -165,6 +168,8 @@ public class RocksDBCheckpointDiffer implements AutoCloseable { private final ScheduledExecutorService executor; private final long maxAllowedTimeInDag; + private final Semaphore bootstrapStateLock = new Semaphore(1); + /** * Constructor. @@ -1056,8 +1061,10 @@ public void pruneOlderSnapshotsWithCompactionHistory() { Set sstFileNodesRemoved = pruneSnapshotFileNodesFromDag(lastCompactionSstFiles); + lockBootstrapState(); removeSstFile(sstFileNodesRemoved); deleteOlderSnapshotsCompactionFiles(olderSnapshotsLogFilePaths); + unlockBootstrapState(); } /** @@ -1356,8 +1363,9 @@ public void pruneSstFiles() { .map(node -> node.getFileName()) .collect(Collectors.toSet()); } - + lockBootstrapState(); removeSstFile(nonLeafSstFiles); + unlockBootstrapState(); } @VisibleForTesting @@ -1374,4 +1382,18 @@ public static Logger getLog() { public ConcurrentHashMap getCompactionNodeMap() { return compactionNodeMap; } + + @Override + public void lockBootstrapState() { + try { + bootstrapStateLock.acquire(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public void unlockBootstrapState() { + bootstrapStateLock.release(); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 3d59eb9384d..5dcaa9ac30b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -33,7 +33,9 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,12 +80,16 @@ * If Kerberos is not enabled, simply append the login user name to * `ozone.administrator`, e.g. `scm` */ -public class OMDBCheckpointServlet extends DBCheckpointServlet { +public class OMDBCheckpointServlet extends DBCheckpointServlet + implements BootstrapStateHandler { private static final Logger LOG = LoggerFactory.getLogger(OMDBCheckpointServlet.class); private static final long serialVersionUID = 1L; private static final String DURATION_TO_WAIT_FOR_DIRECTORY = "PT10S"; + private BootstrapStateHandler keyDeletingService; + private BootstrapStateHandler sstFilteringService; + private BootstrapStateHandler rocksDbCheckpointDiffer; @Override public void init() throws ServletException { @@ -115,6 +121,9 @@ public void init() throws ServletException { allowedUsers, allowedGroups, om.isSpnegoEnabled()); + + keyDeletingService = om.getKeyManager().getDeletingService(); + sstFilteringService = om.getKeyManager().getSnapshotSstFilteringService(); } @Override @@ -278,4 +287,16 @@ private OzoneConfiguration getConf() { .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) .getConfiguration(); } + + @Override + public void lockBootstrapState() throws InterruptedException { + keyDeletingService.lockBootstrapState(); + } + + @Override + public void unlockBootstrapState() { + keyDeletingService.unlockBootstrapState(); + } + + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index 7696c37f21d..a456c2f8e92 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.utils.db.RocksDatabase; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.ozone.rocksdiff.RocksDiffUtils; import org.rocksdb.RocksDBException; @@ -45,6 +46,7 @@ import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -65,7 +67,8 @@ * all the irrelevant and safe to delete sst files that don't correspond * to the bucket on which the snapshot was taken. */ -public class SstFilteringService extends BackgroundService { +public class SstFilteringService extends BackgroundService + implements BootstrapStateHandler{ private static final Logger LOG = LoggerFactory.getLogger(SstFilteringService.class); @@ -101,6 +104,8 @@ public SstFilteringService(long interval, TimeUnit unit, long serviceTimeout, snapshotFilteredCount = new AtomicLong(0); } + private final Semaphore bootstrapStateLock = new Semaphore(1); + private class SstFilteringTask implements BackgroundTask { @Override @@ -150,7 +155,9 @@ public BackgroundTaskResult call() throws Exception { .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), dbName, true); RocksDatabase db = rdbStore.getDb(); + lockBootstrapState(); db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction); + unlockBootstrapState(); // mark the snapshot as filtered by writing to the file String content = snapshotInfo.getSnapshotID() + "\n"; @@ -167,6 +174,8 @@ public BackgroundTaskResult call() throws Exception { return BackgroundTaskResult.EmptyTaskResult.newResult(); } + + /** * @param snapshotInfo * @return a list of pairs (tableName,keyPrefix). @@ -212,4 +221,15 @@ public AtomicLong getSnapshotFilteredCount() { return snapshotFilteredCount; } + + @Override + public void lockBootstrapState() throws InterruptedException { + bootstrapStateLock.acquire(); + } + + @Override + public void unlockBootstrapState() { + bootstrapStateLock.release(); + } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index a769ff711d2..b09b9d2dd26 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.common.DeleteBlockGroupResult; -import org.apache.hadoop.ozone.lock.BootstrapStateLock; +import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; @@ -70,7 +70,7 @@ * keys. */ public class KeyDeletingService extends BackgroundService implements - BootstrapStateLock { + BootstrapStateHandler { private static final Logger LOG = LoggerFactory.getLogger(KeyDeletingService.class); From ba4df8dbeca6992f21e144232f37968ad78372d2 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 7 Feb 2023 17:18:56 -0800 Subject: [PATCH 077/120] only allow single compaction log at a time --- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 +++++++++++++- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 9 +++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 9e6c681a3c4..c7f04fa21e6 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -72,6 +72,9 @@ public class RDBStore implements DBStore { private final RDBMetrics rdbMetrics; private final RocksDBCheckpointDiffer rocksDBCheckpointDiffer; private final String dbJmxBeanName; + private final boolean enableCompactionLog; + private static int compactionLogEnabledCount; + @VisibleForTesting public RDBStore(File dbFile, ManagedDBOptions options, @@ -96,9 +99,15 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, dbLocation = dbFile; dbJmxBeanName = dbJmxBeanNameName == null ? dbFile.getName() : dbJmxBeanNameName; - + this.enableCompactionLog = enableCompactionLog; try { if (enableCompactionLog) { + + compactionLogEnabledCount++; + if (compactionLogEnabledCount > 1) { + throw new RuntimeException("too many compaction logs enabled"); + } + rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer( dbLocation.getParent() + OM_KEY_PREFIX + OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_BACKUP_DIR, OM_COMPACTION_LOG_DIR, dbLocation, maxTimeAllowedForSnapshotInDag, @@ -206,6 +215,9 @@ public void close() throws IOException { statMBeanName = null; } + if (enableCompactionLog) { + compactionLogEnabledCount--; + } RDBMetrics.unRegister(); checkPointManager.close(); db.close(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 7ed188dad79..4d09501696c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -478,11 +478,16 @@ public void start(OzoneConfiguration configuration) throws IOException { public static DBStore loadDB(OzoneConfiguration configuration, File metaDir) throws IOException { - return loadDB(configuration, metaDir, OM_DB_NAME, false); + return loadDB(configuration, metaDir, OM_DB_NAME, false, true); } public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, String dbName, boolean readOnly) throws IOException { + return loadDB(configuration, metaDir, dbName, readOnly, false); + } + + public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, + String dbName, boolean readOnly, boolean enableCompactionLog) throws IOException { final int maxFSSnapshots = configuration.getInt( OZONE_OM_FS_SNAPSHOT_MAX_LIMIT, OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT); @@ -493,7 +498,7 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, .setOpenReadOnly(readOnly) .setPath(Paths.get(metaDir.getPath())) .setMaxFSSnapshots(maxFSSnapshots) - .setEnableCompactionLog(true); + .setEnableCompactionLog(enableCompactionLog); DBStore dbStore = addOMTablesAndCodecs(dbStoreBuilder).build(); return dbStore; } From da44dc26ae73762d81d612386ea512e9e590cb9f Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 7 Feb 2023 17:52:07 -0800 Subject: [PATCH 078/120] updated OMDBCheckpointServlet with bootstrap locks --- .../rocksdiff/RocksDBCheckpointDiffer.java | 22 ++++++++++++------- .../ozone/om/OMDBCheckpointServlet.java | 13 +++++++++-- .../hadoop/ozone/om/SstFilteringService.java | 9 +++++--- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index f4f30ec12ef..1c5e6aa69ff 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -1061,16 +1061,19 @@ public void pruneOlderSnapshotsWithCompactionHistory() { Set sstFileNodesRemoved = pruneSnapshotFileNodesFromDag(lastCompactionSstFiles); - lockBootstrapState(); - removeSstFile(sstFileNodesRemoved); - deleteOlderSnapshotsCompactionFiles(olderSnapshotsLogFilePaths); - unlockBootstrapState(); + try { + lockBootstrapState(); + removeSstFiles(sstFileNodesRemoved); + deleteOlderSnapshotsCompactionFiles(olderSnapshotsLogFilePaths); + } finally { + unlockBootstrapState(); + } } /** * Deletes the SST file from the backup directory if exists. */ - private void removeSstFile(Set sstFileNodes) { + private void removeSstFiles(Set sstFileNodes) { for (String sstFileNode: sstFileNodes) { File file = new File(sstBackupDir + "/" + sstFileNode + SST_FILE_EXTENSION); @@ -1363,9 +1366,12 @@ public void pruneSstFiles() { .map(node -> node.getFileName()) .collect(Collectors.toSet()); } - lockBootstrapState(); - removeSstFile(nonLeafSstFiles); - unlockBootstrapState(); + try { + lockBootstrapState(); + removeSstFiles(nonLeafSstFiles); + } finally { + unlockBootstrapState(); + } } @VisibleForTesting diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 5dcaa9ac30b..8b2ceeca98f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -124,6 +124,8 @@ public void init() throws ServletException { keyDeletingService = om.getKeyManager().getDeletingService(); sstFilteringService = om.getKeyManager().getSnapshotSstFilteringService(); + rocksDbCheckpointDiffer = om.getMetadataManager().getStore() + .getRocksDBCheckpointDiffer(); } @Override @@ -136,14 +138,17 @@ public void writeDbDataToStream(DBCheckpoint checkpoint, // Map of link to path. Map hardLinkFiles = new HashMap<>(); - getFilesForArchive(checkpoint, copyFiles, hardLinkFiles, - includeSnapshotData(request)); try (TarArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(destination)) { + lockBootstrapState(); + getFilesForArchive(checkpoint, copyFiles, hardLinkFiles, + includeSnapshotData(request)); archiveOutputStream .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); writeFilesToArchive(copyFiles, hardLinkFiles, archiveOutputStream); + } finally { + unlockBootstrapState(); } } @@ -291,10 +296,14 @@ private OzoneConfiguration getConf() { @Override public void lockBootstrapState() throws InterruptedException { keyDeletingService.lockBootstrapState(); + sstFilteringService.lockBootstrapState(); + rocksDbCheckpointDiffer.lockBootstrapState(); } @Override public void unlockBootstrapState() { + rocksDbCheckpointDiffer.unlockBootstrapState(); + sstFilteringService.unlockBootstrapState(); keyDeletingService.unlockBootstrapState(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index a456c2f8e92..a5f79728f39 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -155,9 +155,12 @@ public BackgroundTaskResult call() throws Exception { .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), dbName, true); RocksDatabase db = rdbStore.getDb(); - lockBootstrapState(); - db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction); - unlockBootstrapState(); + try { + lockBootstrapState(); + db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction); + } finally { + unlockBootstrapState(); + } // mark the snapshot as filtered by writing to the file String content = snapshotInfo.getSnapshotID() + "\n"; From 993e51cbc9f8529e62e5ac961c439aa739170197 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 8 Feb 2023 10:44:26 -0800 Subject: [PATCH 079/120] moved instance count to differ --- .../apache/hadoop/hdds/utils/db/RDBStore.java | 19 +++++++------------ .../rocksdiff/RocksDBCheckpointDiffer.java | 11 ++++++++++- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index c7f04fa21e6..d8bbbcfdf14 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -72,9 +72,6 @@ public class RDBStore implements DBStore { private final RDBMetrics rdbMetrics; private final RocksDBCheckpointDiffer rocksDBCheckpointDiffer; private final String dbJmxBeanName; - private final boolean enableCompactionLog; - private static int compactionLogEnabledCount; - @VisibleForTesting public RDBStore(File dbFile, ManagedDBOptions options, @@ -99,15 +96,9 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, dbLocation = dbFile; dbJmxBeanName = dbJmxBeanNameName == null ? dbFile.getName() : dbJmxBeanNameName; - this.enableCompactionLog = enableCompactionLog; + try { if (enableCompactionLog) { - - compactionLogEnabledCount++; - if (compactionLogEnabledCount > 1) { - throw new RuntimeException("too many compaction logs enabled"); - } - rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer( dbLocation.getParent() + OM_KEY_PREFIX + OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_BACKUP_DIR, OM_COMPACTION_LOG_DIR, dbLocation, maxTimeAllowedForSnapshotInDag, @@ -215,8 +206,12 @@ public void close() throws IOException { statMBeanName = null; } - if (enableCompactionLog) { - compactionLogEnabledCount--; + if (rocksDBCheckpointDiffer != null) { + try { + rocksDBCheckpointDiffer.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } } RDBMetrics.unRegister(); checkPointManager.close(); diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 1c5e6aa69ff..1efcda3b0ce 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -97,7 +97,6 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private final String sstBackupDir; private final String activeDBLocationStr; - private final String compactionLogDir; /** @@ -170,6 +169,8 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private final long maxAllowedTimeInDag; private final Semaphore bootstrapStateLock = new Semaphore(1); + private static int instanceCount; + private boolean closed = false; /** * Constructor. @@ -191,6 +192,10 @@ public RocksDBCheckpointDiffer(String metadataDir, File activeDBLocation, long maxTimeAllowedForSnapshotInDagInMs, long pruneCompactionDagDaemonRunIntervalInMs) { + instanceCount++; + if (instanceCount > 1 && !metadataDir.contains("integration-test/target/test-dir/MiniOzoneClusterImpl")) { + throw new RuntimeException("too many rocksdb instances"); + } this.compactionLogDir = createCompactionLogDir(metadataDir, compactionLogDirName); this.sstBackupDir = Paths.get(metadataDir, sstBackupDir) + "/"; @@ -311,6 +316,10 @@ public void close() throws Exception { if (executor != null) { executor.shutdown(); } + if (!closed) { + instanceCount--; + closed = true; + } } // Hash table to track CompactionNode for a given SST File. From 80ca9885b9335473af563fffffec194c3d804e04 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 8 Feb 2023 17:00:57 -0800 Subject: [PATCH 080/120] now tests stops metadataManager --- .../ozone/om/ratis/TestOzoneManagerDoubleBuffer.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java index cb977f46db2..44b117fc1de 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java @@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateSnapshotResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; @@ -53,8 +54,7 @@ * This class tests snapshot aware OzoneManagerDoubleBuffer flushing logic. */ class TestOzoneManagerDoubleBuffer { - - private OzoneManagerDoubleBuffer doubleBuffer; + private OzoneManagerDoubleBuffer doubleBuffer; private CreateSnapshotResponse snapshotResponse1 = mock(CreateSnapshotResponse.class); private CreateSnapshotResponse snapshotResponse2 = @@ -73,6 +73,7 @@ class TestOzoneManagerDoubleBuffer { mock(OMBucketCreateResponse.class); @TempDir private File tempDir; + private OMMetadataManager omMetadataManager; @BeforeEach public void setup() throws IOException { @@ -80,7 +81,7 @@ public void setup() throws IOException { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, tempDir.getAbsolutePath()); - OMMetadataManager omMetadataManager = + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration); OzoneManager ozoneManager = mock(OzoneManager.class); when(ozoneManager.getMetrics()).thenReturn(omMetrics); @@ -126,10 +127,11 @@ public void setup() throws IOException { } @AfterEach - public void stop() { + public void stop() throws Exception { if (doubleBuffer != null) { doubleBuffer.stop(); } + omMetadataManager.stop(); } private static Stream doubleBufferFlushCases() { From 1419e98214fbbb80f13b2e6797cd34fcbea9071d Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 9 Feb 2023 17:45:18 -0800 Subject: [PATCH 081/120] checkStyle --- .../ozone/lock/BootstrapStateHandler.java | 24 +++++++++++++++++-- .../hdds/utils/DBCheckpointServlet.java | 3 ++- .../apache/hadoop/hdds/utils/db/RDBStore.java | 4 ++-- .../rocksdiff/RocksDBCheckpointDiffer.java | 11 --------- .../ozone/om/TestOmSnapshotFileSystem.java | 3 ++- .../ozone/om/OMDBCheckpointServlet.java | 12 ++++------ .../ozone/om/OmMetadataManagerImpl.java | 4 ++-- .../hadoop/ozone/om/SstFilteringService.java | 6 ++--- .../ozone/om/TestSstFilteringService.java | 1 - .../ratis/TestOzoneManagerDoubleBuffer.java | 3 +-- .../TestOMSnapshotDeleteResponse.java | 3 --- 11 files changed, 38 insertions(+), 36 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java index c5c6751589c..323b71c11ee 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java @@ -1,6 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package org.apache.hadoop.ozone.lock; +/** + * Manage state that is needed for bootstrapping followers + */ public interface BootstrapStateHandler { - public void lockBootstrapState() throws InterruptedException; - public void unlockBootstrapState(); + void lockBootstrapState() throws InterruptedException; + void unlockBootstrapState(); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 35f3c8ad3be..20737169a5d 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -188,10 +188,11 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { } } } + public void writeDbDataToStream(DBCheckpoint checkpoint, HttpServletRequest ignoredRequest, OutputStream destination) - throws IOException, InterruptedException{ + throws IOException, InterruptedException { writeDBCheckpointToStream(checkpoint, destination); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index d8bbbcfdf14..509bda00ec8 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -101,8 +101,8 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, if (enableCompactionLog) { rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer( dbLocation.getParent() + OM_KEY_PREFIX + OM_SNAPSHOT_DIFF_DIR, - OM_COMPACTION_BACKUP_DIR, OM_COMPACTION_LOG_DIR, dbLocation, maxTimeAllowedForSnapshotInDag, - compactionDagDaemonInterval); + OM_COMPACTION_BACKUP_DIR, OM_COMPACTION_LOG_DIR, dbLocation, + maxTimeAllowedForSnapshotInDag, compactionDagDaemonInterval); rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions); } else { rocksDBCheckpointDiffer = null; diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 1efcda3b0ce..2a55c2ffe11 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -169,9 +169,6 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private final long maxAllowedTimeInDag; private final Semaphore bootstrapStateLock = new Semaphore(1); - private static int instanceCount; - private boolean closed = false; - /** * Constructor. * Note that previous compaction logs are loaded by RDBStore after this @@ -192,10 +189,6 @@ public RocksDBCheckpointDiffer(String metadataDir, File activeDBLocation, long maxTimeAllowedForSnapshotInDagInMs, long pruneCompactionDagDaemonRunIntervalInMs) { - instanceCount++; - if (instanceCount > 1 && !metadataDir.contains("integration-test/target/test-dir/MiniOzoneClusterImpl")) { - throw new RuntimeException("too many rocksdb instances"); - } this.compactionLogDir = createCompactionLogDir(metadataDir, compactionLogDirName); this.sstBackupDir = Paths.get(metadataDir, sstBackupDir) + "/"; @@ -316,10 +309,6 @@ public void close() throws Exception { if (executor != null) { executor.shutdown(); } - if (!closed) { - instanceCount--; - closed = true; - } } // Hash table to track CompactionNode for a given SST File. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java index ae33ca9d10e..7f505b557c7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java @@ -567,7 +567,8 @@ private String createSnapshot() SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager() .getSnapshotInfoTable() .get(SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName)); - String snapshotDirName = getSnapshotPath(conf, snapshotInfo); + String snapshotDirName = getSnapshotPath(conf, snapshotInfo) + + OM_KEY_PREFIX + "CURRENT"; GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), 1000, 120000); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 8b2ceeca98f..c69ea266877 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -21,9 +21,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.compressors.CompressorException; -import org.apache.commons.compress.compressors.CompressorOutputStream; -import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.recon.ReconConfig; @@ -35,7 +32,6 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -122,10 +118,10 @@ public void init() throws ServletException { allowedGroups, om.isSpnegoEnabled()); - keyDeletingService = om.getKeyManager().getDeletingService(); - sstFilteringService = om.getKeyManager().getSnapshotSstFilteringService(); - rocksDbCheckpointDiffer = om.getMetadataManager().getStore() - .getRocksDBCheckpointDiffer(); + keyDeletingService = om.getKeyManager().getDeletingService(); + sstFilteringService = om.getKeyManager().getSnapshotSstFilteringService(); + rocksDbCheckpointDiffer = om.getMetadataManager().getStore() + .getRocksDBCheckpointDiffer(); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 4d09501696c..4fd83cbcdff 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -100,7 +100,6 @@ import static org.apache.hadoop.ozone.OzoneConsts.DB_TRANSIENT_MARKER; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; @@ -487,7 +486,8 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, } public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, - String dbName, boolean readOnly, boolean enableCompactionLog) throws IOException { + String dbName, boolean readOnly, boolean enableCompactionLog) + throws IOException { final int maxFSSnapshots = configuration.getInt( OZONE_OM_FS_SNAPSHOT_MAX_LIMIT, OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index a5f79728f39..96daa458562 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -68,7 +68,7 @@ * to the bucket on which the snapshot was taken. */ public class SstFilteringService extends BackgroundService - implements BootstrapStateHandler{ + implements BootstrapStateHandler { private static final Logger LOG = LoggerFactory.getLogger(SstFilteringService.class); @@ -152,8 +152,8 @@ public BackgroundTaskResult call() throws Exception { String snapshotCheckpointDir = omMetadataDir + OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR; RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl - .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), - dbName, true); + .loadDB(ozoneManager.getConfiguration(), + new File(snapshotCheckpointDir), dbName, true); RocksDatabase db = rdbStore.getDb(); try { lockBootstrapState(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java index c4b8db4a2de..4b0be53a6cc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java @@ -61,7 +61,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java index 44b117fc1de..0a26ff0f9e6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java @@ -35,7 +35,6 @@ import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateSnapshotResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; @@ -54,7 +53,7 @@ * This class tests snapshot aware OzoneManagerDoubleBuffer flushing logic. */ class TestOzoneManagerDoubleBuffer { - private OzoneManagerDoubleBuffer doubleBuffer; + private OzoneManagerDoubleBuffer doubleBuffer; private CreateSnapshotResponse snapshotResponse1 = mock(CreateSnapshotResponse.class); private CreateSnapshotResponse snapshotResponse2 = diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java index 1853c18be04..55f3178365f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java @@ -40,9 +40,6 @@ import java.io.File; import java.util.UUID; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; /** * This class tests OMSnapshotDeleteResponse. From eeb35b963185fb4a50b167929336a53fcafc409f Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 9 Feb 2023 21:27:27 -0800 Subject: [PATCH 082/120] checkstyle --- .../org/apache/hadoop/ozone/lock/BootstrapStateHandler.java | 2 +- .../org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 6 +++--- .../org/apache/hadoop/ozone/om/SstFilteringService.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java index 323b71c11ee..ede3e782a75 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.lock; /** - * Manage state that is needed for bootstrapping followers + * Manage state that is needed for bootstrapping followers. */ public interface BootstrapStateHandler { void lockBootstrapState() throws InterruptedException; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index c69ea266877..ec6d1d68a5b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -83,9 +83,9 @@ public class OMDBCheckpointServlet extends DBCheckpointServlet LoggerFactory.getLogger(OMDBCheckpointServlet.class); private static final long serialVersionUID = 1L; private static final String DURATION_TO_WAIT_FOR_DIRECTORY = "PT10S"; - private BootstrapStateHandler keyDeletingService; - private BootstrapStateHandler sstFilteringService; - private BootstrapStateHandler rocksDbCheckpointDiffer; + private transient BootstrapStateHandler keyDeletingService; + private transient BootstrapStateHandler sstFilteringService; + private transient BootstrapStateHandler rocksDbCheckpointDiffer; @Override public void init() throws ServletException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index 96daa458562..ebe1e07e5f6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -153,7 +153,7 @@ public BackgroundTaskResult call() throws Exception { OM_SNAPSHOT_CHECKPOINT_DIR; RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl .loadDB(ozoneManager.getConfiguration(), - new File(snapshotCheckpointDir), dbName, true); + new File(snapshotCheckpointDir), dbName, true); RocksDatabase db = rdbStore.getDb(); try { lockBootstrapState(); From 3a8589a051aa7ad93c573735087efcd125488ec8 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 10 Feb 2023 11:54:37 -0800 Subject: [PATCH 083/120] initial lock test --- .../ozone/om/TestOMDbCheckpointServlet.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 863885e7697..f1c1ed50d40 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -40,6 +40,10 @@ import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -53,6 +57,7 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.security.UserGroupInformation; @@ -83,6 +88,7 @@ import org.junit.rules.TemporaryFolder; import org.junit.rules.Timeout; import org.mockito.Matchers; +import org.mockito.Mockito; import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE; import static org.apache.hadoop.ozone.om.OmSnapshotManager.truncateFileName; @@ -90,7 +96,9 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -400,6 +408,46 @@ public void testWriteDbDataWithoutOmSnapshot() Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); } + @Test + public void testBootstrapLocking() + throws Exception { + setupCluster(); + BootstrapStateHandler keyDeletingService = + cluster.getOzoneManager().getKeyManager() + .getDeletingService(); + BootstrapStateHandler sstFilteringService = + cluster.getOzoneManager().getKeyManager() + .getSnapshotSstFilteringService(); + BootstrapStateHandler differ = + cluster.getOzoneManager().getMetadataManager() + .getStore().getRocksDBCheckpointDiffer(); + ExecutorService executorService = Executors.newCachedThreadPool(); + + OMDBCheckpointServlet omDbCheckpointServlet = new OMDBCheckpointServlet(); + + OMDBCheckpointServlet spyServlet = Mockito.spy(omDbCheckpointServlet); + ServletContext servletContext = mock(ServletContext.class); + when(servletContext.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) + .thenReturn(cluster.getOzoneManager()); + doReturn(servletContext).when(spyServlet).getServletContext(); + + + spyServlet.init(); + + spyServlet.lockBootstrapState(); + Future testJob = executorService.submit(() -> { + try { + keyDeletingService.lockBootstrapState(); + keyDeletingService.unlockBootstrapState(); + } catch (InterruptedException e) { + } + }); + Assert.assertThrows(TimeoutException.class, + () -> testJob.get(5000, TimeUnit.MILLISECONDS)); + executorService.shutdownNow(); + + } + private void prepSnapshotData() throws Exception { From 2e76dd833036d4d54c85137d7ec03a568ac7f45e Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 10 Feb 2023 13:07:59 -0800 Subject: [PATCH 084/120] more lock tests --- .../ozone/om/TestOMDbCheckpointServlet.java | 55 ++++++++++++++++--- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index f1c1ed50d40..5c18a40122f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -411,7 +411,11 @@ public void testWriteDbDataWithoutOmSnapshot() @Test public void testBootstrapLocking() throws Exception { - setupCluster(); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(1) + .build(); + cluster.waitForClusterToBeReady(); + BootstrapStateHandler keyDeletingService = cluster.getOzoneManager().getKeyManager() .getDeletingService(); @@ -435,20 +439,53 @@ public void testBootstrapLocking() spyServlet.init(); spyServlet.lockBootstrapState(); - Future testJob = executorService.submit(() -> { - try { - keyDeletingService.lockBootstrapState(); - keyDeletingService.unlockBootstrapState(); - } catch (InterruptedException e) { - } - }); + Future kdsTest = checkLock(keyDeletingService, executorService); Assert.assertThrows(TimeoutException.class, - () -> testJob.get(5000, TimeUnit.MILLISECONDS)); + () -> kdsTest.get(500, TimeUnit.MILLISECONDS)); + Future filterTest = checkLock(sstFilteringService, executorService); + Assert.assertThrows(TimeoutException.class, + () -> filterTest.get(500, TimeUnit.MILLISECONDS)); + Future differTest = checkLock(differ, executorService); + Assert.assertThrows(TimeoutException.class, + () -> differTest.get(500, TimeUnit.MILLISECONDS)); + + spyServlet.unlockBootstrapState(); + keyDeletingService.lockBootstrapState(); + Future servletTest1 = checkLock(spyServlet, executorService); + Assert.assertThrows(TimeoutException.class, + () ->servletTest1.get(500, TimeUnit.MILLISECONDS)); + + keyDeletingService.unlockBootstrapState(); + sstFilteringService.lockBootstrapState(); + Future servletTest2 = checkLock(spyServlet, executorService); + Assert.assertThrows(TimeoutException.class, + () ->servletTest2.get(500, TimeUnit.MILLISECONDS)); + sstFilteringService.unlockBootstrapState(); + differ.lockBootstrapState(); + + Future servletTest3 = checkLock(spyServlet, executorService); + Assert.assertThrows(TimeoutException.class, + () ->servletTest3.get(500, TimeUnit.MILLISECONDS)); + + differ.unlockBootstrapState(); + Future servletTest4 = checkLock(spyServlet, executorService); + Assert.assertNull(servletTest4.get(500, TimeUnit.MILLISECONDS)); + executorService.shutdownNow(); } + private Future checkLock(BootstrapStateHandler handler, + ExecutorService executorService) { + return executorService.submit(() -> { + try { + handler.lockBootstrapState(); + handler.unlockBootstrapState(); + } catch (InterruptedException e) { + } + }); + } private void prepSnapshotData() throws Exception { setupCluster(); From cc292510856cecbd0f37bba2f2eeb88b7c7ad743 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 10 Feb 2023 13:27:40 -0800 Subject: [PATCH 085/120] clean up --- .../ozone/om/TestOMDbCheckpointServlet.java | 74 ++++++++++--------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 5c18a40122f..a3ad3b2d677 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -409,16 +409,12 @@ public void testWriteDbDataWithoutOmSnapshot() } @Test - public void testBootstrapLocking() - throws Exception { - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(1) - .build(); + public void testBootstrapLocking() throws Exception { + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build(); cluster.waitForClusterToBeReady(); BootstrapStateHandler keyDeletingService = - cluster.getOzoneManager().getKeyManager() - .getDeletingService(); + cluster.getOzoneManager().getKeyManager().getDeletingService(); BootstrapStateHandler sstFilteringService = cluster.getOzoneManager().getKeyManager() .getSnapshotSstFilteringService(); @@ -429,7 +425,7 @@ public void testBootstrapLocking() OMDBCheckpointServlet omDbCheckpointServlet = new OMDBCheckpointServlet(); - OMDBCheckpointServlet spyServlet = Mockito.spy(omDbCheckpointServlet); + OMDBCheckpointServlet spyServlet = spy(omDbCheckpointServlet); ServletContext servletContext = mock(ServletContext.class); when(servletContext.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) .thenReturn(cluster.getOzoneManager()); @@ -438,36 +434,25 @@ public void testBootstrapLocking() spyServlet.init(); + // Confirm the other processes are locked out when the bootstrap + // servlet takes the lock. spyServlet.lockBootstrapState(); - Future kdsTest = checkLock(keyDeletingService, executorService); - Assert.assertThrows(TimeoutException.class, - () -> kdsTest.get(500, TimeUnit.MILLISECONDS)); - Future filterTest = checkLock(sstFilteringService, executorService); - Assert.assertThrows(TimeoutException.class, - () -> filterTest.get(500, TimeUnit.MILLISECONDS)); - Future differTest = checkLock(differ, executorService); - Assert.assertThrows(TimeoutException.class, - () -> differTest.get(500, TimeUnit.MILLISECONDS)); - + confirmServletLocksOutOtherHandler(keyDeletingService, executorService); + confirmServletLocksOutOtherHandler(sstFilteringService, executorService); + confirmServletLocksOutOtherHandler(differ, executorService); spyServlet.unlockBootstrapState(); - keyDeletingService.lockBootstrapState(); - Future servletTest1 = checkLock(spyServlet, executorService); - Assert.assertThrows(TimeoutException.class, - () ->servletTest1.get(500, TimeUnit.MILLISECONDS)); - keyDeletingService.unlockBootstrapState(); - sstFilteringService.lockBootstrapState(); - Future servletTest2 = checkLock(spyServlet, executorService); - Assert.assertThrows(TimeoutException.class, - () ->servletTest2.get(500, TimeUnit.MILLISECONDS)); - sstFilteringService.unlockBootstrapState(); - differ.lockBootstrapState(); - - Future servletTest3 = checkLock(spyServlet, executorService); - Assert.assertThrows(TimeoutException.class, - () ->servletTest3.get(500, TimeUnit.MILLISECONDS)); - - differ.unlockBootstrapState(); + // Confirm the bootstrap servlet is locked out when any of the other + // proecesses takes the lock. + confirmOtherHandlerLocksOutServlet(keyDeletingService, spyServlet, + executorService); + confirmOtherHandlerLocksOutServlet(sstFilteringService, spyServlet, + executorService); + confirmOtherHandlerLocksOutServlet(differ, spyServlet, + executorService); + + // Confirm that servlet takes the look when none of the other + // processes have it. Future servletTest4 = checkLock(spyServlet, executorService); Assert.assertNull(servletTest4.get(500, TimeUnit.MILLISECONDS)); @@ -475,6 +460,25 @@ public void testBootstrapLocking() } + private void confirmServletLocksOutOtherHandler(BootstrapStateHandler handler, + ExecutorService executorService) { + Future test = checkLock(handler, executorService); + // handler should fail to take the lock because the servlet has taken theirs + Assert.assertThrows(TimeoutException.class, + () -> test.get(500, TimeUnit.MILLISECONDS)); + } + + private void confirmOtherHandlerLocksOutServlet(BootstrapStateHandler handler, + BootstrapStateHandler servlet, ExecutorService executorService) + throws InterruptedException { + handler.lockBootstrapState(); + Future test = checkLock(servlet, executorService); + // servlet should fail to lock when other handler has taken theirs + Assert.assertThrows(TimeoutException.class, + () -> test.get(500, TimeUnit.MILLISECONDS)); + handler.unlockBootstrapState(); + } + private Future checkLock(BootstrapStateHandler handler, ExecutorService executorService) { return executorService.submit(() -> { From eb4002cf7a3c395d0ebffe211f0f8288b054493a Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 10 Feb 2023 13:49:42 -0800 Subject: [PATCH 086/120] checkstyle --- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index a3ad3b2d677..115a4a65f77 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -88,7 +88,6 @@ import org.junit.rules.TemporaryFolder; import org.junit.rules.Timeout; import org.mockito.Matchers; -import org.mockito.Mockito; import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE; import static org.apache.hadoop.ozone.om.OmSnapshotManager.truncateFileName; @@ -443,7 +442,7 @@ public void testBootstrapLocking() throws Exception { spyServlet.unlockBootstrapState(); // Confirm the bootstrap servlet is locked out when any of the other - // proecesses takes the lock. + // processes takes the lock. confirmOtherHandlerLocksOutServlet(keyDeletingService, spyServlet, executorService); confirmOtherHandlerLocksOutServlet(sstFilteringService, spyServlet, @@ -451,10 +450,10 @@ public void testBootstrapLocking() throws Exception { confirmOtherHandlerLocksOutServlet(differ, spyServlet, executorService); - // Confirm that servlet takes the look when none of the other + // Confirm that servlet takes the lock when none of the other // processes have it. - Future servletTest4 = checkLock(spyServlet, executorService); - Assert.assertNull(servletTest4.get(500, TimeUnit.MILLISECONDS)); + Future servletTest = checkLock(spyServlet, executorService); + Assert.assertNull(servletTest.get(500, TimeUnit.MILLISECONDS)); executorService.shutdownNow(); From 62c4f025a1fde21bcc22567c777d3b835c508be1 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 10 Feb 2023 14:40:08 -0800 Subject: [PATCH 087/120] split the bootstrap state changes into separate PR. --- .../ozone/lock/BootstrapStateHandler.java | 26 ------ .../apache/hadoop/hdds/utils/db/RDBStore.java | 7 -- .../rocksdiff/RocksDBCheckpointDiffer.java | 40 ++------- .../ozone/om/TestOMDbCheckpointServlet.java | 88 ------------------- .../apache/hadoop/ozone/om/KeyManager.java | 5 +- .../hadoop/ozone/om/KeyManagerImpl.java | 8 +- .../ozone/om/OMDBCheckpointServlet.java | 35 +------- .../ozone/om/OmMetadataManagerImpl.java | 10 +-- .../hadoop/ozone/om/SstFilteringService.java | 31 +------ .../ozone/om/service/KeyDeletingService.java | 16 +--- .../ratis/TestOzoneManagerDoubleBuffer.java | 7 +- 11 files changed, 26 insertions(+), 247 deletions(-) delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java deleted file mode 100644 index ede3e782a75..00000000000 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/BootstrapStateHandler.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.lock; - -/** - * Manage state that is needed for bootstrapping followers. - */ -public interface BootstrapStateHandler { - void lockBootstrapState() throws InterruptedException; - void unlockBootstrapState(); -} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 509bda00ec8..aeb7eb5eb57 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -206,13 +206,6 @@ public void close() throws IOException { statMBeanName = null; } - if (rocksDBCheckpointDiffer != null) { - try { - rocksDBCheckpointDiffer.close(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } RDBMetrics.unRegister(); checkPointManager.close(); db.close(); diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 2a55c2ffe11..a8eb7904598 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -24,11 +24,9 @@ import java.util.Collections; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.rocksdb.AbstractEventListener; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; @@ -89,14 +87,14 @@ * It is important to note that compaction log is per-DB instance. Since * each OM DB instance might trigger compactions at different timings. */ -public class RocksDBCheckpointDiffer implements AutoCloseable, - BootstrapStateHandler { +public class RocksDBCheckpointDiffer implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(RocksDBCheckpointDiffer.class); private final String sstBackupDir; private final String activeDBLocationStr; + private final String compactionLogDir; /** @@ -167,7 +165,6 @@ public class RocksDBCheckpointDiffer implements AutoCloseable, private final ScheduledExecutorService executor; private final long maxAllowedTimeInDag; - private final Semaphore bootstrapStateLock = new Semaphore(1); /** * Constructor. @@ -1059,19 +1056,14 @@ public void pruneOlderSnapshotsWithCompactionHistory() { Set sstFileNodesRemoved = pruneSnapshotFileNodesFromDag(lastCompactionSstFiles); - try { - lockBootstrapState(); - removeSstFiles(sstFileNodesRemoved); - deleteOlderSnapshotsCompactionFiles(olderSnapshotsLogFilePaths); - } finally { - unlockBootstrapState(); - } + removeSstFile(sstFileNodesRemoved); + deleteOlderSnapshotsCompactionFiles(olderSnapshotsLogFilePaths); } /** * Deletes the SST file from the backup directory if exists. */ - private void removeSstFiles(Set sstFileNodes) { + private void removeSstFile(Set sstFileNodes) { for (String sstFileNode: sstFileNodes) { File file = new File(sstBackupDir + "/" + sstFileNode + SST_FILE_EXTENSION); @@ -1364,12 +1356,8 @@ public void pruneSstFiles() { .map(node -> node.getFileName()) .collect(Collectors.toSet()); } - try { - lockBootstrapState(); - removeSstFiles(nonLeafSstFiles); - } finally { - unlockBootstrapState(); - } + + removeSstFile(nonLeafSstFiles); } @VisibleForTesting @@ -1386,18 +1374,4 @@ public static Logger getLog() { public ConcurrentHashMap getCompactionNodeMap() { return compactionNodeMap; } - - @Override - public void lockBootstrapState() { - try { - bootstrapStateLock.acquire(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public void unlockBootstrapState() { - bootstrapStateLock.release(); - } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 115a4a65f77..863885e7697 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -40,10 +40,6 @@ import java.util.List; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -57,7 +53,6 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.security.UserGroupInformation; @@ -95,9 +90,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -407,88 +400,7 @@ public void testWriteDbDataWithoutOmSnapshot() Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); } - @Test - public void testBootstrapLocking() throws Exception { - cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build(); - cluster.waitForClusterToBeReady(); - - BootstrapStateHandler keyDeletingService = - cluster.getOzoneManager().getKeyManager().getDeletingService(); - BootstrapStateHandler sstFilteringService = - cluster.getOzoneManager().getKeyManager() - .getSnapshotSstFilteringService(); - BootstrapStateHandler differ = - cluster.getOzoneManager().getMetadataManager() - .getStore().getRocksDBCheckpointDiffer(); - ExecutorService executorService = Executors.newCachedThreadPool(); - - OMDBCheckpointServlet omDbCheckpointServlet = new OMDBCheckpointServlet(); - - OMDBCheckpointServlet spyServlet = spy(omDbCheckpointServlet); - ServletContext servletContext = mock(ServletContext.class); - when(servletContext.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) - .thenReturn(cluster.getOzoneManager()); - doReturn(servletContext).when(spyServlet).getServletContext(); - - - spyServlet.init(); - - // Confirm the other processes are locked out when the bootstrap - // servlet takes the lock. - spyServlet.lockBootstrapState(); - confirmServletLocksOutOtherHandler(keyDeletingService, executorService); - confirmServletLocksOutOtherHandler(sstFilteringService, executorService); - confirmServletLocksOutOtherHandler(differ, executorService); - spyServlet.unlockBootstrapState(); - - // Confirm the bootstrap servlet is locked out when any of the other - // processes takes the lock. - confirmOtherHandlerLocksOutServlet(keyDeletingService, spyServlet, - executorService); - confirmOtherHandlerLocksOutServlet(sstFilteringService, spyServlet, - executorService); - confirmOtherHandlerLocksOutServlet(differ, spyServlet, - executorService); - - // Confirm that servlet takes the lock when none of the other - // processes have it. - Future servletTest = checkLock(spyServlet, executorService); - Assert.assertNull(servletTest.get(500, TimeUnit.MILLISECONDS)); - - executorService.shutdownNow(); - } - - private void confirmServletLocksOutOtherHandler(BootstrapStateHandler handler, - ExecutorService executorService) { - Future test = checkLock(handler, executorService); - // handler should fail to take the lock because the servlet has taken theirs - Assert.assertThrows(TimeoutException.class, - () -> test.get(500, TimeUnit.MILLISECONDS)); - } - - private void confirmOtherHandlerLocksOutServlet(BootstrapStateHandler handler, - BootstrapStateHandler servlet, ExecutorService executorService) - throws InterruptedException { - handler.lockBootstrapState(); - Future test = checkLock(servlet, executorService); - // servlet should fail to lock when other handler has taken theirs - Assert.assertThrows(TimeoutException.class, - () -> test.get(500, TimeUnit.MILLISECONDS)); - handler.unlockBootstrapState(); - } - - private Future checkLock(BootstrapStateHandler handler, - ExecutorService executorService) { - return executorService.submit(() -> { - try { - handler.lockBootstrapState(); - handler.unlockBootstrapState(); - } catch (InterruptedException e) { - } - }); - - } private void prepSnapshotData() throws Exception { setupCluster(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index e60f0882a73..c14e97ea2b5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -28,7 +28,6 @@ import org.apache.hadoop.ozone.om.fs.OzoneManagerFS; import org.apache.hadoop.hdds.utils.BackgroundService; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; -import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket; import java.io.IOException; @@ -155,7 +154,7 @@ List getExpiredOpenKeys(Duration expireThreshold, int count, * Returns the instance of Deleting Service. * @return Background service. */ - KeyDeletingService getDeletingService(); + BackgroundService getDeletingService(); OmMultipartUploadList listMultipartUploads(String volumeName, @@ -248,5 +247,5 @@ List getPendingDeletionSubFiles(long volumeId, * Returns the instance of Snapshot SST Filtering service. * @return Background service. */ - SstFilteringService getSnapshotSstFilteringService(); + BackgroundService getSnapshotSstFilteringService(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 25c33e0550f..92e96c85e1f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -157,9 +157,9 @@ public class KeyManagerImpl implements KeyManager { private final OzoneBlockTokenSecretManager secretManager; private final boolean grpcBlockTokenEnabled; - private KeyDeletingService keyDeletingService; + private BackgroundService keyDeletingService; - private SstFilteringService snapshotSstFilteringService; + private BackgroundService snapshotSstFilteringService; private final KeyProviderCryptoExtension kmsProvider; private final boolean enableFileSystemPaths; @@ -605,7 +605,7 @@ public OMMetadataManager getMetadataManager() { } @Override - public KeyDeletingService getDeletingService() { + public BackgroundService getDeletingService() { return keyDeletingService; } @@ -618,7 +618,7 @@ public BackgroundService getOpenKeyCleanupService() { return openKeyCleanupService; } - public SstFilteringService getSnapshotSstFilteringService() { + public BackgroundService getSnapshotSstFilteringService() { return snapshotSstFilteringService; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index ec6d1d68a5b..1264d5b78f2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; @@ -76,16 +75,12 @@ * If Kerberos is not enabled, simply append the login user name to * `ozone.administrator`, e.g. `scm` */ -public class OMDBCheckpointServlet extends DBCheckpointServlet - implements BootstrapStateHandler { +public class OMDBCheckpointServlet extends DBCheckpointServlet { private static final Logger LOG = LoggerFactory.getLogger(OMDBCheckpointServlet.class); private static final long serialVersionUID = 1L; private static final String DURATION_TO_WAIT_FOR_DIRECTORY = "PT10S"; - private transient BootstrapStateHandler keyDeletingService; - private transient BootstrapStateHandler sstFilteringService; - private transient BootstrapStateHandler rocksDbCheckpointDiffer; @Override public void init() throws ServletException { @@ -117,11 +112,6 @@ public void init() throws ServletException { allowedUsers, allowedGroups, om.isSpnegoEnabled()); - - keyDeletingService = om.getKeyManager().getDeletingService(); - sstFilteringService = om.getKeyManager().getSnapshotSstFilteringService(); - rocksDbCheckpointDiffer = om.getMetadataManager().getStore() - .getRocksDBCheckpointDiffer(); } @Override @@ -134,17 +124,14 @@ public void writeDbDataToStream(DBCheckpoint checkpoint, // Map of link to path. Map hardLinkFiles = new HashMap<>(); + getFilesForArchive(checkpoint, copyFiles, hardLinkFiles, + includeSnapshotData(request)); try (TarArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(destination)) { - lockBootstrapState(); - getFilesForArchive(checkpoint, copyFiles, hardLinkFiles, - includeSnapshotData(request)); archiveOutputStream .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); writeFilesToArchive(copyFiles, hardLinkFiles, archiveOutputStream); - } finally { - unlockBootstrapState(); } } @@ -288,20 +275,4 @@ private OzoneConfiguration getConf() { .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) .getConfiguration(); } - - @Override - public void lockBootstrapState() throws InterruptedException { - keyDeletingService.lockBootstrapState(); - sstFilteringService.lockBootstrapState(); - rocksDbCheckpointDiffer.lockBootstrapState(); - } - - @Override - public void unlockBootstrapState() { - rocksDbCheckpointDiffer.unlockBootstrapState(); - sstFilteringService.unlockBootstrapState(); - keyDeletingService.unlockBootstrapState(); - } - - } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 91860d5b646..6820b851e8b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -480,17 +480,11 @@ public void start(OzoneConfiguration configuration) throws IOException { public static DBStore loadDB(OzoneConfiguration configuration, File metaDir) throws IOException { - return loadDB(configuration, metaDir, OM_DB_NAME, false, true); + return loadDB(configuration, metaDir, OM_DB_NAME, false); } public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, String dbName, boolean readOnly) throws IOException { - return loadDB(configuration, metaDir, dbName, readOnly, false); - } - - public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, - String dbName, boolean readOnly, boolean enableCompactionLog) - throws IOException { final int maxFSSnapshots = configuration.getInt( OZONE_OM_FS_SNAPSHOT_MAX_LIMIT, OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT); @@ -501,7 +495,7 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, .setOpenReadOnly(readOnly) .setPath(Paths.get(metaDir.getPath())) .setMaxFSSnapshots(maxFSSnapshots) - .setEnableCompactionLog(enableCompactionLog); + .setEnableCompactionLog(true); DBStore dbStore = addOMTablesAndCodecs(dbStoreBuilder).build(); return dbStore; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index ebe1e07e5f6..7696c37f21d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hdds.utils.db.RocksDatabase; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.ozone.rocksdiff.RocksDiffUtils; import org.rocksdb.RocksDBException; @@ -46,7 +45,6 @@ import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -67,8 +65,7 @@ * all the irrelevant and safe to delete sst files that don't correspond * to the bucket on which the snapshot was taken. */ -public class SstFilteringService extends BackgroundService - implements BootstrapStateHandler { +public class SstFilteringService extends BackgroundService { private static final Logger LOG = LoggerFactory.getLogger(SstFilteringService.class); @@ -104,8 +101,6 @@ public SstFilteringService(long interval, TimeUnit unit, long serviceTimeout, snapshotFilteredCount = new AtomicLong(0); } - private final Semaphore bootstrapStateLock = new Semaphore(1); - private class SstFilteringTask implements BackgroundTask { @Override @@ -152,15 +147,10 @@ public BackgroundTaskResult call() throws Exception { String snapshotCheckpointDir = omMetadataDir + OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR; RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl - .loadDB(ozoneManager.getConfiguration(), - new File(snapshotCheckpointDir), dbName, true); + .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), + dbName, true); RocksDatabase db = rdbStore.getDb(); - try { - lockBootstrapState(); - db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction); - } finally { - unlockBootstrapState(); - } + db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction); // mark the snapshot as filtered by writing to the file String content = snapshotInfo.getSnapshotID() + "\n"; @@ -177,8 +167,6 @@ public BackgroundTaskResult call() throws Exception { return BackgroundTaskResult.EmptyTaskResult.newResult(); } - - /** * @param snapshotInfo * @return a list of pairs (tableName,keyPrefix). @@ -224,15 +212,4 @@ public AtomicLong getSnapshotFilteredCount() { return snapshotFilteredCount; } - - @Override - public void lockBootstrapState() throws InterruptedException { - bootstrapStateLock.acquire(); - } - - @Override - public void unlockBootstrapState() { - bootstrapStateLock.release(); - } - } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index b09b9d2dd26..b0c6b7a13ad 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -31,7 +30,6 @@ import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.common.DeleteBlockGroupResult; -import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; @@ -69,8 +67,7 @@ * metadata accordingly, if scm returns success for keys, then clean up those * keys. */ -public class KeyDeletingService extends BackgroundService implements - BootstrapStateHandler { +public class KeyDeletingService extends BackgroundService { private static final Logger LOG = LoggerFactory.getLogger(KeyDeletingService.class); @@ -86,7 +83,6 @@ public class KeyDeletingService extends BackgroundService implements private final int keyLimitPerTask; private final AtomicLong deletedKeyCount; private final AtomicLong runCount; - private final Semaphore bootstrapStateLock = new Semaphore(1); public KeyDeletingService(OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient, @@ -145,16 +141,6 @@ private boolean isRatisEnabled() { return ozoneManager.isRatisEnabled(); } - @Override - public void lockBootstrapState() throws InterruptedException { - bootstrapStateLock.acquire(); - } - - @Override - public void unlockBootstrapState() { - bootstrapStateLock.release(); - } - /** * A key deleting task scans OM DB and looking for a certain number of * pending-deletion keys, sends these keys along with their associated blocks diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java index 0a26ff0f9e6..cb977f46db2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java @@ -53,6 +53,7 @@ * This class tests snapshot aware OzoneManagerDoubleBuffer flushing logic. */ class TestOzoneManagerDoubleBuffer { + private OzoneManagerDoubleBuffer doubleBuffer; private CreateSnapshotResponse snapshotResponse1 = mock(CreateSnapshotResponse.class); @@ -72,7 +73,6 @@ class TestOzoneManagerDoubleBuffer { mock(OMBucketCreateResponse.class); @TempDir private File tempDir; - private OMMetadataManager omMetadataManager; @BeforeEach public void setup() throws IOException { @@ -80,7 +80,7 @@ public void setup() throws IOException { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, tempDir.getAbsolutePath()); - omMetadataManager = + OMMetadataManager omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration); OzoneManager ozoneManager = mock(OzoneManager.class); when(ozoneManager.getMetrics()).thenReturn(omMetrics); @@ -126,11 +126,10 @@ public void setup() throws IOException { } @AfterEach - public void stop() throws Exception { + public void stop() { if (doubleBuffer != null) { doubleBuffer.stop(); } - omMetadataManager.stop(); } private static Stream doubleBufferFlushCases() { From 76796ff557443b6d260512aa76f57a48a486ee9f Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 10 Feb 2023 14:43:11 -0800 Subject: [PATCH 088/120] checkstyle --- .../java/org/apache/hadoop/ozone/om/SstFilteringService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index 7696c37f21d..86098534f80 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -147,8 +147,8 @@ public BackgroundTaskResult call() throws Exception { String snapshotCheckpointDir = omMetadataDir + OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR; RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl - .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), - dbName, true); + .loadDB(ozoneManager.getConfiguration(), + new File(snapshotCheckpointDir), dbName, true); RocksDatabase db = rdbStore.getDb(); db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction); From d7a66bc17de59451a834fc907c0336291b17c900 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 16 Feb 2023 11:50:00 -0800 Subject: [PATCH 089/120] checkstyle --- .../test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java | 1 - .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 5 ----- 2 files changed, 6 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java index 997e506f1ab..cde97212726 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java @@ -69,7 +69,6 @@ import java.util.stream.Collectors; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.CONTAINS_SNAPSHOT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 3817ebe4aef..8988739e460 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -24,11 +24,6 @@ import com.google.common.cache.LoadingCache; import com.google.common.cache.RemovalListener; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.concurrent.ExecutionException; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.RDBStore; From b8384c4d81e5634ff222c93fdf1157f795f17899 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 16 Feb 2023 12:13:53 -0800 Subject: [PATCH 090/120] fix directory --- .../org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java index dcbdebf9bf0..eb5b62e1c65 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java @@ -50,6 +50,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; /** * Test client-side CRUD snapshot operations with Ozone Manager. @@ -326,16 +327,14 @@ private String createSnapshot() throws Exception { // Asserts that create request succeeded Assertions.assertEquals(0, res); - File metaDir = OMStorage - .getOmDbDir(ozoneManager.getConfiguration()); + OzoneConfiguration conf = ozoneManager.getConfiguration(); // wait till the snapshot directory exists SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager() .getSnapshotInfoTable() .get(SnapshotInfo.getTableKey(VOLUME, BUCKET, snapshotName)); - String snapshotDirName = metaDir + OM_KEY_PREFIX + - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + OM_DB_NAME + - snapshotInfo.getCheckpointDirName() + OM_KEY_PREFIX + "CURRENT"; + String snapshotDirName = getSnapshotPath(conf, snapshotInfo) + + OM_KEY_PREFIX + "CURRENT"; GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), 1000, 100000); From ed069f01142452dfc59b3f9703e486ef0434d7e4 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 16 Feb 2023 13:09:15 -0800 Subject: [PATCH 091/120] checkstyle --- .../java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java index eb5b62e1c65..5ca8fdbcb51 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java @@ -29,7 +29,6 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.util.ToolRunner; @@ -45,9 +44,7 @@ import org.junit.jupiter.params.provider.ValueSource; import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; From ab15ef03b2de5337e5acf29c50d6e424845f2936 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Thu, 16 Mar 2023 15:37:06 -0700 Subject: [PATCH 092/120] merge cleanup --- .../ozone/om/TestOMDbCheckpointServlet.java | 4 ++-- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 17 +++++++++++++++++ .../hadoop/ozone/om/OmSnapshotManager.java | 6 ------ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 53df1679e01..0108f05c23a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -406,7 +406,7 @@ private void prepSnapshotData() throws Exception { setupCluster(); metaDir = OMStorage.getOmDbDir(conf); - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster.newClient()); // Create dummy keys for snapshotting. TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), @@ -461,7 +461,7 @@ private String createSnapshot(String vname, String bname) throws IOException, InterruptedException, TimeoutException { final OzoneManager om = cluster.getOzoneManager(); String snapshotName = UUID.randomUUID().toString(); - OzoneManagerProtocol writeClient = cluster.getRpcClient().getObjectStore() + OzoneManagerProtocol writeClient = cluster.newClient().getObjectStore() .getClientProxy().getOzoneManagerClient(); writeClient.createSnapshot(vname, bname, snapshotName); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 2337c13b43c..1b8061eb2d9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -340,6 +340,23 @@ public static OmMetadataManagerImpl createCheckpointMetadataManager( return new OmMetadataManagerImpl(conf, dir, name); } + /** + * Metadata constructor for checkpoints. + * + * @param conf - Ozone conf. + * @param dir - Checkpoint parent directory. + * @param name - Checkpoint directory name. + * @throws IOException + */ + private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) + throws IOException { + lock = new OmReadOnlyLock(); + omEpoch = 0; + setStore(loadDB(conf, dir, name, true)); + initializeOmTables(false); + } + + // metadata constructor for snapshots OmMetadataManagerImpl(OzoneConfiguration conf, String snapshotDirName, boolean isSnapshotInCache) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 6e0c4cfe94e..ea63ebe5486 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -591,11 +591,6 @@ private static Path getFullPath(Path dbPath, String fileName) { return Paths.get(dbPath.getParent().toString(), fileName); } - - private ManagedRocksDB createDbForSnapshotDiff(OzoneConfiguration config) { - final ManagedOptions managedOptions = new ManagedOptions(); - managedOptions.setCreateIfMissing(true); -======= private int getIndexFromToken(final String token) throws IOException { if (isBlank(token)) { return 0; @@ -615,7 +610,6 @@ private int getIndexFromToken(final String token) throws IOException { "Resend the request with valid token returned in previous request."); } } ->>>>>>> origin/master private ManagedRocksDB createRocksDbForSnapshotDiff( final ManagedDBOptions dbOptions, String dbPath, From fc6549e77da97a7c1c22d9fcd3bcbe4038cf457d Mon Sep 17 00:00:00 2001 From: GeorgeJahad Date: Fri, 17 Mar 2023 14:53:48 -0700 Subject: [PATCH 093/120] Apply suggestions from code review Co-authored-by: Siyao Meng <50227127+smengcl@users.noreply.github.com> --- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 9 ++++----- .../org/apache/hadoop/ozone/om/OmSnapshotManager.java | 2 -- .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 4 ++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 0108f05c23a..f6b3888f9cd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -313,9 +313,8 @@ public void testWriteDbDataToStream() throws Exception { FileUtil.unTar(tempFile, newDbDir); // Move snapshot dir to correct location. - new File(newDbDirName, OM_SNAPSHOT_DIR) - .renameTo(new File(newDbDir.getParent(), OM_SNAPSHOT_DIR)); - + Assert.assertTrue(new File(newDbDirName, OM_SNAPSHOT_DIR) + .renameTo(new File(newDbDir.getParent(), OM_SNAPSHOT_DIR))); // Confirm the checkpoint directories match, (after remove extras). Path checkpointLocation = dbCheckpoint.getCheckpointLocation(); @@ -427,8 +426,8 @@ private void prepSnapshotData() throws Exception { new File(snapshotDirName).getParent(), "fabricatedSnapshot"); fabricatedSnapshot.toFile().mkdirs(); - Paths.get(fabricatedSnapshot.toString(), "fabricatedFile") - .toFile().createNewFile(); + Assert.assertTrue(Paths.get(fabricatedSnapshot.toString(), "fabricatedFile") + .toFile().createNewFile()); // Create fabricated links to snapshot dirs // to confirm that links are recognized even if diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index ea63ebe5486..c86f3ecca30 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -577,8 +577,6 @@ static void createHardLinks(Path dbPath) throws IOException { } } } - - // Prepend the full path to the hard link entry entry. @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private static Path getFullPath(Path dbPath, String fileName) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 9b8bafa9310..4ba3b21c4d0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3787,8 +3787,8 @@ File replaceOMDBWithCheckpoint(long lastAppliedIndex, File oldDB, } private void moveCheckpointFiles(File oldDB, Path checkpointPath, File dbDir, - File dbBackup, File dbSnapshotsDir, - File dbSnapshotsBackup) throws IOException { + File dbBackup, File dbSnapshotsDir, + File dbSnapshotsBackup) throws IOException { // Move the new DB checkpoint into the om metadata dir Path markerFile = new File(dbDir, DB_TRANSIENT_MARKER).toPath(); try { From 8c32c5be1b5d396b8b15231a9dc9a16837bfc1de Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 17 Mar 2023 11:24:11 -0700 Subject: [PATCH 094/120] checkstyle --- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 3 ++- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 3 ++- .../apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 3 --- .../apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 2 -- .../org/apache/hadoop/ozone/om/OmSnapshotManager.java | 11 ----------- .../apache/hadoop/ozone/om/SstFilteringService.java | 4 ++-- .../hadoop/ozone/om/ratis_snapshot/package-info.java | 2 +- 7 files changed, 7 insertions(+), 21 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 2b965d467f4..2e0aab81145 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -101,7 +101,8 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, if (enableCompactionLog) { rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer( dbLocation.getParent() + OM_KEY_PREFIX + OM_SNAPSHOT_DIFF_DIR, - OM_COMPACTION_BACKUP_DIR, OM_COMPACTION_LOG_DIR, dbLocation.toString(), + OM_COMPACTION_BACKUP_DIR, OM_COMPACTION_LOG_DIR, + dbLocation.toString(), maxTimeAllowedForSnapshotInDag, compactionDagDaemonInterval); rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions); } else { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index f6b3888f9cd..c4f6c0e34df 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -405,7 +405,8 @@ private void prepSnapshotData() throws Exception { setupCluster(); metaDir = OMStorage.getOmDbDir(conf); - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster.newClient()); + OzoneBucket bucket = TestDataUtil + .createVolumeAndBucket(cluster.newClient()); // Create dummy keys for snapshotting. TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 1264d5b78f2..315e7191dca 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.lang.StringUtils; @@ -200,7 +199,6 @@ private void waitForDirToExist(Path dir) } } - @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE") private void processDir(Path dir, Map copyFiles, Map hardLinkFiles, Set snapshotPaths) @@ -243,7 +241,6 @@ private boolean includeSnapshotData(HttpServletRequest request) { return false; } - @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private void writeFilesToArchive(Map copyFiles, Map hardLinkFiles, ArchiveOutputStream archiveOutputStream) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 1b8061eb2d9..1ecdb55c6c4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -35,7 +35,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import com.google.common.base.Optional; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -331,7 +330,6 @@ protected OmMetadataManagerImpl() { this.omEpoch = 0; } - @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) public static OmMetadataManagerImpl createCheckpointMetadataManager( OzoneConfiguration conf, DBCheckpoint checkpoint) throws IOException { Path path = checkpoint.getCheckpointLocation(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index c86f3ecca30..1c2734445be 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -24,7 +24,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.cache.RemovalListener; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.File; import java.io.IOException; import java.nio.file.Paths; @@ -53,10 +52,6 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; @@ -71,12 +66,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; -import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; @@ -518,7 +509,6 @@ private void verifySnapshotInfoForSnapDiff(final SnapshotInfo fromSnapshot, * @param hardLinkFiles - Map of link->file paths. * @return Path to the file of links created. */ - @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) static Path createHardLinkList(int truncateLength, Map hardLinkFiles) throws IOException { @@ -578,7 +568,6 @@ static void createHardLinks(Path dbPath) throws IOException { } } // Prepend the full path to the hard link entry entry. - @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}) private static Path getFullPath(Path dbPath, String fileName) { File file = new File(fileName); // If there is no directory then this file belongs in the db. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index c2728e53087..b070cbb95b1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -147,8 +147,8 @@ public BackgroundTaskResult call() throws Exception { String snapshotCheckpointDir = omMetadataDir + OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR; try (RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl - .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), - dbName, true)) { + .loadDB(ozoneManager.getConfiguration(), + new File(snapshotCheckpointDir), dbName, true)) { RocksDatabase db = rdbStore.getDb(); db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/package-info.java index 20d0058c579..fce79c0a1f2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/package-info.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/package-info.java @@ -20,4 +20,4 @@ /** * This package contains OM Ratis Snapshot related classes. - */ \ No newline at end of file + */ From f0e790d3b7d39de8b86fb00be845b86a9889b654 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 17 Mar 2023 15:00:13 -0700 Subject: [PATCH 095/120] findbugs --- .../apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 315e7191dca..4323ef9cfff 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -205,9 +205,11 @@ private void processDir(Path dir, Map copyFiles, throws IOException { try (Stream files = Files.list(dir)) { for (Path file : files.collect(Collectors.toList())) { - if (file.toFile().isDirectory()) { + File f = file.toFile(); + if (f.isDirectory()) { // Skip any unexpected snapshot files. - if (file.getParent().toString().endsWith(OM_SNAPSHOT_CHECKPOINT_DIR) + String parent = f.getParent(); + if (parent != null && parent.endsWith(OM_SNAPSHOT_CHECKPOINT_DIR) && !snapshotPaths.contains(file)) { continue; } @@ -254,7 +256,10 @@ private void writeFilesToArchive(Map copyFiles, String fixedFile = truncateFileName(truncateLength, file); if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { // checkpoint files go to root of tarball - fixedFile = Paths.get(fixedFile).getFileName().toString(); + Path f = Paths.get(fixedFile).getFileName(); + if (f != null) { + fixedFile = f.toString(); + } } includeFile(file.toFile(), fixedFile, archiveOutputStream); } From a24709efcd17b98f090027d9cbfd15c1faab01d1 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 17 Mar 2023 15:53:15 -0700 Subject: [PATCH 096/120] findbugs --- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 12 +++++++++--- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 14 +++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 1ecdb55c6c4..bd3a2c9216f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -333,9 +333,15 @@ protected OmMetadataManagerImpl() { public static OmMetadataManagerImpl createCheckpointMetadataManager( OzoneConfiguration conf, DBCheckpoint checkpoint) throws IOException { Path path = checkpoint.getCheckpointLocation(); - File dir = path.getParent().toFile(); - String name = path.getFileName().toString(); - return new OmMetadataManagerImpl(conf, dir, name); + try { + Path parent = path.getParent(); + File dir = parent.toFile(); + String name = path.getFileName().toString(); + return new OmMetadataManagerImpl(conf, dir, name); + // NPE check forced by findbugs NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE + } catch (NullPointerException e) { + throw new IOException("Invalid database: " + checkpoint.getCheckpointLocation()); + } } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 1c2734445be..1767121cf10 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -518,7 +518,10 @@ static Path createHardLinkList(int truncateLength, String fixedFile = truncateFileName(truncateLength, entry.getValue()); // If this file is from the active db, strip the path. if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { - fixedFile = Paths.get(fixedFile).getFileName().toString(); + Path f = Paths.get(fixedFile).getFileName(); + if (f != null) { + fixedFile = f.toString(); + } } sb.append(truncateFileName(truncateLength, entry.getKey())) .append("\t") @@ -568,14 +571,19 @@ static void createHardLinks(Path dbPath) throws IOException { } } // Prepend the full path to the hard link entry entry. - private static Path getFullPath(Path dbPath, String fileName) { + private static Path getFullPath(Path dbPath, String fileName) + throws IOException { File file = new File(fileName); // If there is no directory then this file belongs in the db. if (file.getName().equals(fileName)) { return Paths.get(dbPath.toString(), fileName); } // Else this file belong in a directory parallel to the db. - return Paths.get(dbPath.getParent().toString(), fileName); + Path parent = dbPath.getParent(); + if (parent == null) { + throw new IOException("Invalid database " + dbPath); + } + return Paths.get(parent.toString(), fileName); } private int getIndexFromToken(final String token) throws IOException { From b68479650f64f5e10ee75f518f382d56eaf01ee5 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 20 Mar 2023 10:35:33 -0700 Subject: [PATCH 097/120] fixed streams --- .../ozone/om/TestOMDbCheckpointServlet.java | 33 ++++++++----------- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 4 +-- .../hadoop/ozone/om/OmSnapshotManager.java | 30 +++++++++-------- 3 files changed, 32 insertions(+), 35 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index c4f6c0e34df..444deb5bb0a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -341,22 +341,22 @@ public void testWriteDbDataToStream() throws Exception { getFiles(Paths.get(testDirName, OM_SNAPSHOT_DIR), testDirLength); // Check each line in the hard link file. - Stream lines = Files.lines(Paths.get(newDbDirName, - OM_HARDLINK_FILE)); - List fabricatedLinkLines = new ArrayList<>(); - for (String line: lines.collect(Collectors.toList())) { - Assert.assertFalse("CURRENT file is not a hard link", - line.contains("CURRENT")); - if (line.contains("fabricatedFile")) { - fabricatedLinkLines.add(line); - } else { - checkLine(shortSnapshotLocation, shortSnapshotLocation2, line); - // add links to the final set - finalFullSet.add(line.split("\t")[0]); + try (Stream lines = Files.lines(Paths.get(newDbDirName, + OM_HARDLINK_FILE))) { + + for (String line : lines.collect(Collectors.toList())) { + Assert.assertFalse("CURRENT file is not a hard link", + line.contains("CURRENT")); + if (line.contains("fabricatedFile")) { + fabricatedLinkLines.add(line); + } else { + checkLine(shortSnapshotLocation, shortSnapshotLocation2, line); + // add links to the final set + finalFullSet.add(line.split("\t")[0]); + } } } - Set directories = Sets.newHashSet( shortSnapshotLocation, shortSnapshotLocation2, shortCompactionDirLocation); @@ -364,7 +364,7 @@ public void testWriteDbDataToStream() throws Exception { Set initialFullSet = getFiles(Paths.get(metaDir.toString(), OM_SNAPSHOT_DIR), metaDirLength); - Assert.assertEquals("found expected snapshot files", + Assert.assertEquals("expected snapshot files not found", initialFullSet, finalFullSet); } @@ -399,8 +399,6 @@ public void testWriteDbDataWithoutOmSnapshot() Assert.assertEquals(initialCheckpointSet, finalCheckpointSet); } - - private void prepSnapshotData() throws Exception { setupCluster(); metaDir = OMStorage.getOmDbDir(conf); @@ -414,9 +412,6 @@ private void prepSnapshotData() throws Exception { TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), "content"); - // This sleep can be removed after this is fixed: - // https://issues.apache.org/jira/browse/HDDS-7279 - Thread.sleep(2000); snapshotDirName = createSnapshot(bucket.getVolumeName(), bucket.getName()); snapshotDirName2 = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 88d12354edd..7c41b7e57ec 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -171,7 +171,7 @@ public void testInstallSnapshot() throws Exception { // Do some transactions so that the log index increases List keys = writeKeysToIncreaseLogIndex(leaderRatisServer, 200); - createSnapshot(leaderOM, keys); + createOzoneSnapshot(leaderOM, keys); // Get the latest db checkpoint from the leader OM. TransactionInfo transactionInfo = @@ -563,7 +563,7 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { Assert.assertTrue(logCapture.getOutput().contains(msg)); } - private void createSnapshot(OzoneManager leaderOM, List keys) + private void createOzoneSnapshot(OzoneManager leaderOM, List keys) throws TimeoutException, InterruptedException, IOException { // Avoid double buffer issue waiting for keys. GenericTestUtils.waitFor(() -> { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 1767121cf10..c8ad02d3448 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -68,6 +68,7 @@ import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.util.Map; +import java.util.stream.Stream; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; @@ -553,20 +554,21 @@ static void createHardLinks(Path dbPath) throws IOException { OM_HARDLINK_FILE); if (hardLinkFile.exists()) { // Read file. - List lines = - Files.lines(hardLinkFile.toPath()).collect(Collectors.toList()); - - // Create a link for each line. - for (String l : lines) { - String from = l.split("\t")[1]; - String to = l.split("\t")[0]; - Path fullFromPath = getFullPath(dbPath, from); - Path fullToPath = getFullPath(dbPath, to); - Files.createLink(fullToPath, fullFromPath); - } - if (!hardLinkFile.delete()) { - throw new IOException( - "Failed to delete: " + hardLinkFile); + try (Stream s = Files.lines(hardLinkFile.toPath())) { + List lines = s.collect(Collectors.toList()); + + // Create a link for each line. + for (String l : lines) { + String from = l.split("\t")[1]; + String to = l.split("\t")[0]; + Path fullFromPath = getFullPath(dbPath, from); + Path fullToPath = getFullPath(dbPath, to); + Files.createLink(fullToPath, fullFromPath); + } + if (!hardLinkFile.delete()) { + throw new IOException( + "Failed to delete: " + hardLinkFile); + } } } } From 9f814282f9e1a9fabe4c9da2d5021696a27538eb Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 20 Mar 2023 11:01:15 -0700 Subject: [PATCH 098/120] fix merge --- .../java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 43d9be455e3..71e01347678 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -356,7 +356,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) throws IOException { lock = new OmReadOnlyLock(); omEpoch = 0; - setStore(loadDB(conf, dir, name, true)); + setStore(loadDB(conf, dir, name, true, java.util.Optional.of(Boolean.TRUE))); initializeOmTables(false); } From 7c60c53e32a14715a9b77ce627efaa0176058bb4 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 20 Mar 2023 12:41:36 -0700 Subject: [PATCH 099/120] now uses RDBCheckpointManager.waitForCheckpointDirectoryExist --- .../hadoop/hdds/utils/db/RDBCheckpointManager.java | 4 +++- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 13 ++++--------- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 3 ++- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java index ee5408b13b9..1651553af1b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java @@ -113,7 +113,7 @@ public RocksDBCheckpoint createCheckpoint(String parentDir, String name) { * Wait for checkpoint directory to be created for 5 secs with 100 millis * poll interval. */ - public static void waitForCheckpointDirectoryExist(File file) + public static boolean waitForCheckpointDirectoryExist(File file) throws IOException { Instant start = Instant.now(); try { @@ -126,9 +126,11 @@ public static void waitForCheckpointDirectoryExist(File file) " availability.", Duration.between(start, Instant.now()).toMillis(), file.getAbsoluteFile()); + return true; } catch (ConditionTimeoutException exception) { LOG.info("Checkpoint directory: {} didn't get created in 5 secs.", file.getAbsolutePath()); + return false; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 4323ef9cfff..4efd785c66f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.DBCheckpointServlet; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; +import org.apache.hadoop.hdds.utils.db.RDBCheckpointManager; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneConsts; @@ -187,15 +188,9 @@ private Set waitForSnapshotDirs(DBCheckpoint checkpoint) return snapshotPaths; } - private void waitForDirToExist(Path dir) - throws IOException, InterruptedException { - long endTime = System.currentTimeMillis() + - Duration.parse(DURATION_TO_WAIT_FOR_DIRECTORY).toMillis(); - while (!dir.toFile().exists()) { - Thread.sleep(100); - if (System.currentTimeMillis() > endTime) { - throw new IOException("snapshot dir doesn't exist: " + dir); - } + private void waitForDirToExist(Path dir) throws IOException { + if (!RDBCheckpointManager.waitForCheckpointDirectoryExist(dir.toFile())) { + throw new IOException("snapshot dir doesn't exist: " + dir); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 71e01347678..93b5f2f0fa7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -356,7 +356,8 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) throws IOException { lock = new OmReadOnlyLock(); omEpoch = 0; - setStore(loadDB(conf, dir, name, true, java.util.Optional.of(Boolean.TRUE))); + setStore(loadDB(conf, dir, name, true, + java.util.Optional.of(Boolean.TRUE))); initializeOmTables(false); } From c06b404a8857e09633dfdbdffb7fa0083b394190 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 20 Mar 2023 15:59:33 -0700 Subject: [PATCH 100/120] more hard link tests --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 85 ++++++++++++------- 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 7c41b7e57ec..161439f4f35 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; +import org.apache.hadoop.hdds.utils.db.RDBCheckpointManager; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.client.BucketArgs; @@ -55,7 +56,9 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -63,8 +66,14 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithData.createKey; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -171,7 +180,7 @@ public void testInstallSnapshot() throws Exception { // Do some transactions so that the log index increases List keys = writeKeysToIncreaseLogIndex(leaderRatisServer, 200); - createOzoneSnapshot(leaderOM, keys); + SnapshotInfo snapshotInfo = createOzoneSnapshot(leaderOM, keys); // Get the latest db checkpoint from the leader OM. TransactionInfo transactionInfo = @@ -253,6 +262,35 @@ public void testInstallSnapshot() throws Exception { omKeyInfo = followerOM.lookupKey(omKeyArgs); Assertions.assertNotNull(omKeyInfo); Assertions.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); + + // Confirm followers snapshot hard links are as expected + File followerMetaDir = OMStorage.getOmDbDir(followerOM.getConfiguration()); + Path followerActiveDir = Paths.get(followerMetaDir.toString(), OM_DB_NAME); + Path followerSnapshotDir = + Paths.get(getSnapshotPath(followerOM.getConfiguration(), snapshotInfo)); + Path leaderSnapshotDir = + Paths.get(getSnapshotPath(leaderOM.getConfiguration(), snapshotInfo)); + // Check snapshot hard links + + // Get the list of sst files from the leader. Then use those filenames to + // compare the followers snapshot with the followers active fs for + // hard links + int sstCount = 0; + try (Streamlist = Files.list(leaderSnapshotDir)) { + for (Path p: list.collect(Collectors.toList())) { + String fileName = p.getFileName().toString(); + if (fileName.toLowerCase().endsWith(".sst")) { + Path snapshotSST = Paths.get(followerSnapshotDir.toString(), fileName); + Path activeSST = Paths.get(followerActiveDir.toString(), fileName); + assertEquals("Snapshot sst file is supposed to be a hard link", + OmSnapshotManager.getINode(activeSST), + OmSnapshotManager.getINode(snapshotSST)); + sstCount++; + + } + } + } + assertTrue("No sst files were found", sstCount > 0); } @Ignore("Enable this unit test after RATIS-1481 used") @@ -563,36 +601,25 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { Assert.assertTrue(logCapture.getOutput().contains(msg)); } - private void createOzoneSnapshot(OzoneManager leaderOM, List keys) - throws TimeoutException, InterruptedException, IOException { - // Avoid double buffer issue waiting for keys. - GenericTestUtils.waitFor(() -> { - try { - OmKeyInfo key = leaderOM.getMetadataManager() - .getKeyTable(ozoneBucket.getBucketLayout()) - .getSkipCache(leaderOM.getMetadataManager() - .getOzoneKey(volumeName, bucketName, keys.get(0))); - return key != null; - } catch (Exception e) { - return false; - } - }, 100, 10000); + private SnapshotInfo createOzoneSnapshot(OzoneManager leaderOM, List keys) + throws IOException { objectStore.createSnapshot(volumeName, bucketName, "snap1"); - // Allow the snapshot to be written to the info table. - GenericTestUtils.waitFor(() -> { - try { - String tableKey = SnapshotInfo.getTableKey(volumeName, - bucketName, - "snap1"); - SnapshotInfo snapshotInfo = leaderOM.getMetadataManager() - .getSnapshotInfoTable() - .getSkipCache(tableKey); - return snapshotInfo != null; - } catch (Exception e) { - return false; - } - }, 100, 3000); + String tableKey = SnapshotInfo.getTableKey(volumeName, + bucketName, + "snap1"); + SnapshotInfo snapshotInfo = leaderOM.getMetadataManager() + .getSnapshotInfoTable() + .get(tableKey); + // Allow the snapshot to be written to disk + String fileName = + getSnapshotPath(leaderOM.getConfiguration(), snapshotInfo); + File snapshotDir = new File(fileName); + if (!RDBCheckpointManager + .waitForCheckpointDirectoryExist(snapshotDir)) { + throw new IOException("snapshot directory doesn't exist"); + } + return snapshotInfo; } private List writeKeysToIncreaseLogIndex( From 9f93f5084b55308cb4e5031d439236c54856acc7 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 21 Mar 2023 09:11:10 -0700 Subject: [PATCH 101/120] parameterize snapshot e2e test --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 161439f4f35..b805bf1caca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -51,6 +51,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import org.slf4j.event.Level; @@ -160,8 +162,9 @@ public void shutdown() { } } - @Test - public void testInstallSnapshot() throws Exception { + @ParameterizedTest + @ValueSource(ints = {1, 10}) + public void testInstallSnapshot(int numSnapshotsToCreate) throws Exception { // Get the leader OM String leaderOMNodeId = OmFailoverProxyUtil .getFailoverProxyProvider(objectStore.getClientProxy()) @@ -178,9 +181,18 @@ public void testInstallSnapshot() throws Exception { OzoneManager followerOM = cluster.getOzoneManager(followerNodeId); // Do some transactions so that the log index increases - List keys = writeKeysToIncreaseLogIndex(leaderRatisServer, 200); - - SnapshotInfo snapshotInfo = createOzoneSnapshot(leaderOM, keys); + int keyIncrement = 200; + int keyCount = 0; + String snapshotNamePrefix = "snapshot"; + String snapshotName = ""; + List keys = new ArrayList<>(); + SnapshotInfo snapshotInfo = null; + for (int snapshotCount = 0; snapshotCount < numSnapshotsToCreate; snapshotCount++) { + keyCount += keyIncrement; + snapshotName = snapshotNamePrefix + snapshotCount; + keys = writeKeysToIncreaseLogIndex(leaderRatisServer, keyCount); + snapshotInfo = createOzoneSnapshot(leaderOM, snapshotName); + } // Get the latest db checkpoint from the leader OM. TransactionInfo transactionInfo = @@ -198,11 +210,11 @@ public void testInstallSnapshot() throws Exception { // The recently started OM should be lagging behind the leader OM. // Wait & for follower to update transactions to leader snapshot index. - // Timeout error if follower does not load update within 3s + // Timeout error if follower does not load update within 10s GenericTestUtils.waitFor(() -> { return followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex() >= leaderOMSnapshotIndex - 1; - }, 100, 3000); + }, 100, 10000); long followerOMLastAppliedIndex = followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex(); @@ -253,12 +265,12 @@ public void testInstallSnapshot() throws Exception { volumeName, bucketName, newKeys.get(0)))); */ - // Read back data from the OM snapshot. + // Read back data from the last OM snapshot. OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) .setBucketName(bucketName) - .setKeyName(".snapshot/snap1/" + keys.get(0)).build(); - OmKeyInfo omKeyInfo = null; + .setKeyName(".snapshot/"+ snapshotName + "/" + keys.get(0)).build(); + OmKeyInfo omKeyInfo; omKeyInfo = followerOM.lookupKey(omKeyArgs); Assertions.assertNotNull(omKeyInfo); Assertions.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); @@ -601,13 +613,14 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { Assert.assertTrue(logCapture.getOutput().contains(msg)); } - private SnapshotInfo createOzoneSnapshot(OzoneManager leaderOM, List keys) + private SnapshotInfo createOzoneSnapshot(OzoneManager leaderOM, + String snapshotName) throws IOException { - objectStore.createSnapshot(volumeName, bucketName, "snap1"); + objectStore.createSnapshot(volumeName, bucketName, snapshotName); String tableKey = SnapshotInfo.getTableKey(volumeName, bucketName, - "snap1"); + snapshotName); SnapshotInfo snapshotInfo = leaderOM.getMetadataManager() .getSnapshotInfoTable() .get(tableKey); From 4f4c77a1974f04f14b3ed931f1a7ed511bfcfadc Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 21 Mar 2023 11:18:43 -0700 Subject: [PATCH 102/120] fix test for restructured directory --- .../om/snapshot/TestOzoneManagerSnapshotAcl.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java index 9a14eb0e979..b2eb4bf72a4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java @@ -29,6 +29,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; +import org.apache.hadoop.hdds.utils.db.RDBCheckpointManager; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.BucketArgs; @@ -70,6 +71,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.junit.jupiter.params.provider.Arguments.arguments; /** @@ -479,11 +481,14 @@ private void createSnapshot() .getMetadataManager() .getSnapshotInfoTable() .get(SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName)); - final String snapshotDirName = metaDir + OM_KEY_PREFIX + - OM_SNAPSHOT_DIR + OM_KEY_PREFIX + OM_DB_NAME + - snapshotInfo.getCheckpointDirName() + OM_KEY_PREFIX + "CURRENT"; - GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), - 1000, 120000); + // Allow the snapshot to be written to disk + String fileName = + getSnapshotPath(ozoneManager.getConfiguration(), snapshotInfo); + File snapshotDir = new File(fileName); + if (!RDBCheckpointManager + .waitForCheckpointDirectoryExist(snapshotDir)) { + throw new IOException("snapshot directory doesn't exist"); + } } private void setBucketAcl() throws IOException { From 68bbbd1b0992417a239de55c4cf10db8f98f8795 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 21 Mar 2023 13:27:47 -0700 Subject: [PATCH 103/120] added parameterized tests --- .../apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 11 +++++++---- .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index b805bf1caca..17ccb76fcbb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -162,9 +162,9 @@ public void shutdown() { } } - @ParameterizedTest - @ValueSource(ints = {1, 10}) - public void testInstallSnapshot(int numSnapshotsToCreate) throws Exception { + @Test + public void testInstallSnapshot() throws Exception { + int numSnapshotsToCreate = 10; // Get the leader OM String leaderOMNodeId = OmFailoverProxyUtil .getFailoverProxyProvider(objectStore.getClientProxy()) @@ -190,7 +190,7 @@ public void testInstallSnapshot(int numSnapshotsToCreate) throws Exception { for (int snapshotCount = 0; snapshotCount < numSnapshotsToCreate; snapshotCount++) { keyCount += keyIncrement; snapshotName = snapshotNamePrefix + snapshotCount; - keys = writeKeysToIncreaseLogIndex(leaderRatisServer, keyCount); + keys = writeKeys(keyIncrement); snapshotInfo = createOzoneSnapshot(leaderOM, snapshotName); } @@ -294,6 +294,9 @@ public void testInstallSnapshot(int numSnapshotsToCreate) throws Exception { if (fileName.toLowerCase().endsWith(".sst")) { Path snapshotSST = Paths.get(followerSnapshotDir.toString(), fileName); Path activeSST = Paths.get(followerActiveDir.toString(), fileName); + if (!activeSST.toFile().exists()) { + throw new RuntimeException("gbjfix"); + } assertEquals("Snapshot sst file is supposed to be a hard link", OmSnapshotManager.getINode(activeSST), OmSnapshotManager.getINode(snapshotSST)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 2c230d56bfd..16a5bc558a4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3803,8 +3803,8 @@ private void moveCheckpointFiles(File oldDB, Path checkpointPath, File dbDir, Files.deleteIfExists(markerFile); } catch (IOException e) { LOG.error("Failed to move downloaded DB checkpoint {} to metadata " + - "directory {}. Resetting to original DB.", checkpointPath, - oldDB.toPath()); + "directory {}. Resetting to original DB. IOException: {}", checkpointPath, + oldDB.toPath(), e); try { Files.move(dbBackup.toPath(), oldDB.toPath()); if (dbSnapshotsBackup.exists()) { From 068a1bb240d427123aaf380a6854099e37dd2709 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 21 Mar 2023 13:30:59 -0700 Subject: [PATCH 104/120] removed parameterized test --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 38 ++++++------------- .../apache/hadoop/ozone/om/OzoneManager.java | 4 +- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 17ccb76fcbb..161439f4f35 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -51,8 +51,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import org.slf4j.event.Level; @@ -164,7 +162,6 @@ public void shutdown() { @Test public void testInstallSnapshot() throws Exception { - int numSnapshotsToCreate = 10; // Get the leader OM String leaderOMNodeId = OmFailoverProxyUtil .getFailoverProxyProvider(objectStore.getClientProxy()) @@ -181,18 +178,9 @@ public void testInstallSnapshot() throws Exception { OzoneManager followerOM = cluster.getOzoneManager(followerNodeId); // Do some transactions so that the log index increases - int keyIncrement = 200; - int keyCount = 0; - String snapshotNamePrefix = "snapshot"; - String snapshotName = ""; - List keys = new ArrayList<>(); - SnapshotInfo snapshotInfo = null; - for (int snapshotCount = 0; snapshotCount < numSnapshotsToCreate; snapshotCount++) { - keyCount += keyIncrement; - snapshotName = snapshotNamePrefix + snapshotCount; - keys = writeKeys(keyIncrement); - snapshotInfo = createOzoneSnapshot(leaderOM, snapshotName); - } + List keys = writeKeysToIncreaseLogIndex(leaderRatisServer, 200); + + SnapshotInfo snapshotInfo = createOzoneSnapshot(leaderOM, keys); // Get the latest db checkpoint from the leader OM. TransactionInfo transactionInfo = @@ -210,11 +198,11 @@ public void testInstallSnapshot() throws Exception { // The recently started OM should be lagging behind the leader OM. // Wait & for follower to update transactions to leader snapshot index. - // Timeout error if follower does not load update within 10s + // Timeout error if follower does not load update within 3s GenericTestUtils.waitFor(() -> { return followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex() >= leaderOMSnapshotIndex - 1; - }, 100, 10000); + }, 100, 3000); long followerOMLastAppliedIndex = followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex(); @@ -265,12 +253,12 @@ public void testInstallSnapshot() throws Exception { volumeName, bucketName, newKeys.get(0)))); */ - // Read back data from the last OM snapshot. + // Read back data from the OM snapshot. OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) .setBucketName(bucketName) - .setKeyName(".snapshot/"+ snapshotName + "/" + keys.get(0)).build(); - OmKeyInfo omKeyInfo; + .setKeyName(".snapshot/snap1/" + keys.get(0)).build(); + OmKeyInfo omKeyInfo = null; omKeyInfo = followerOM.lookupKey(omKeyArgs); Assertions.assertNotNull(omKeyInfo); Assertions.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); @@ -294,9 +282,6 @@ public void testInstallSnapshot() throws Exception { if (fileName.toLowerCase().endsWith(".sst")) { Path snapshotSST = Paths.get(followerSnapshotDir.toString(), fileName); Path activeSST = Paths.get(followerActiveDir.toString(), fileName); - if (!activeSST.toFile().exists()) { - throw new RuntimeException("gbjfix"); - } assertEquals("Snapshot sst file is supposed to be a hard link", OmSnapshotManager.getINode(activeSST), OmSnapshotManager.getINode(snapshotSST)); @@ -616,14 +601,13 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { Assert.assertTrue(logCapture.getOutput().contains(msg)); } - private SnapshotInfo createOzoneSnapshot(OzoneManager leaderOM, - String snapshotName) + private SnapshotInfo createOzoneSnapshot(OzoneManager leaderOM, List keys) throws IOException { - objectStore.createSnapshot(volumeName, bucketName, snapshotName); + objectStore.createSnapshot(volumeName, bucketName, "snap1"); String tableKey = SnapshotInfo.getTableKey(volumeName, bucketName, - snapshotName); + "snap1"); SnapshotInfo snapshotInfo = leaderOM.getMetadataManager() .getSnapshotInfoTable() .get(tableKey); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 16a5bc558a4..2c230d56bfd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3803,8 +3803,8 @@ private void moveCheckpointFiles(File oldDB, Path checkpointPath, File dbDir, Files.deleteIfExists(markerFile); } catch (IOException e) { LOG.error("Failed to move downloaded DB checkpoint {} to metadata " + - "directory {}. Resetting to original DB. IOException: {}", checkpointPath, - oldDB.toPath(), e); + "directory {}. Resetting to original DB.", checkpointPath, + oldDB.toPath()); try { Files.move(dbBackup.toPath(), oldDB.toPath()); if (dbSnapshotsBackup.exists()) { From 6e4c317dde6f54a46bdc5c1ab5534ad89e23eb6c Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 21 Mar 2023 14:06:12 -0700 Subject: [PATCH 105/120] now compares hardlinks with leader --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 161439f4f35..5982afb8fa0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -268,29 +268,37 @@ public void testInstallSnapshot() throws Exception { Path followerActiveDir = Paths.get(followerMetaDir.toString(), OM_DB_NAME); Path followerSnapshotDir = Paths.get(getSnapshotPath(followerOM.getConfiguration(), snapshotInfo)); + File leaderMetaDir = OMStorage.getOmDbDir(leaderOM.getConfiguration()); + Path leaderActiveDir = Paths.get(leaderMetaDir.toString(), OM_DB_NAME); Path leaderSnapshotDir = Paths.get(getSnapshotPath(leaderOM.getConfiguration(), snapshotInfo)); // Check snapshot hard links - // Get the list of sst files from the leader. Then use those filenames to - // compare the followers snapshot with the followers active fs for - // hard links - int sstCount = 0; + // Get the list of hardlinks from the leader. Then confirm those links + // are on the follower + int hardLinkCount = 0; try (Streamlist = Files.list(leaderSnapshotDir)) { for (Path p: list.collect(Collectors.toList())) { String fileName = p.getFileName().toString(); if (fileName.toLowerCase().endsWith(".sst")) { - Path snapshotSST = Paths.get(followerSnapshotDir.toString(), fileName); - Path activeSST = Paths.get(followerActiveDir.toString(), fileName); - assertEquals("Snapshot sst file is supposed to be a hard link", - OmSnapshotManager.getINode(activeSST), - OmSnapshotManager.getINode(snapshotSST)); - sstCount++; + + Path leaderSnapshotSST = Paths.get(leaderSnapshotDir.toString(), fileName); + Path leaderActiveSST = Paths.get(leaderActiveDir.toString(), fileName); + // If it is a hard link on the leader, it should be a hard link on the follower + if (OmSnapshotManager.getINode(leaderActiveSST) + .equals(OmSnapshotManager.getINode(leaderSnapshotSST))) { + Path followerSnapshotSST = Paths.get(followerSnapshotDir.toString(), fileName); + Path followerActiveSST = Paths.get(followerActiveDir.toString(), fileName); + assertEquals("Snapshot sst file is supposed to be a hard link", + OmSnapshotManager.getINode(followerActiveSST), + OmSnapshotManager.getINode(followerSnapshotSST)); + hardLinkCount++; + } } } } - assertTrue("No sst files were found", sstCount > 0); + assertTrue("No hard links were found", hardLinkCount > 0); } @Ignore("Enable this unit test after RATIS-1481 used") From bc8352613b0799f1a7b59ef6cac8a4a9ecc6125c Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 21 Mar 2023 14:31:03 -0700 Subject: [PATCH 106/120] fixed test to only check leader --- .../hadoop/hdds/utils/DBCheckpointServlet.java | 9 +++++++++ .../apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 12 ++++++++---- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 7 +++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 20737169a5d..618c752f314 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -189,6 +189,15 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { } } + /** + * Write checkpoint to the stream + * + * @param checkpoint The checkpoint to be written. + * @param ignoredRequest The httpRequest which generated this checkpoint + * (Parameter is ignored in this class but used in child classes). + * @param destination The stream to write to. + */ + @Override public void writeDbDataToStream(DBCheckpoint checkpoint, HttpServletRequest ignoredRequest, OutputStream destination) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 5982afb8fa0..8a4f07194b8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -278,12 +278,16 @@ public void testInstallSnapshot() throws Exception { // are on the follower int hardLinkCount = 0; try (Streamlist = Files.list(leaderSnapshotDir)) { - for (Path p: list.collect(Collectors.toList())) { - String fileName = p.getFileName().toString(); + for (Path leaderSnapshotSST: list.collect(Collectors.toList())) { + String fileName = leaderSnapshotSST.getFileName().toString(); if (fileName.toLowerCase().endsWith(".sst")) { - Path leaderSnapshotSST = Paths.get(leaderSnapshotDir.toString(), fileName); - Path leaderActiveSST = Paths.get(leaderActiveDir.toString(), fileName); + Path leaderActiveSST = + Paths.get(leaderActiveDir.toString(), fileName); + // Skip if not hard link on the leader + if (!leaderActiveSST.toFile().exists()) { + continue; + } // If it is a hard link on the leader, it should be a hard link on the follower if (OmSnapshotManager.getINode(leaderActiveSST) .equals(OmSnapshotManager.getINode(leaderSnapshotSST))) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 4efd785c66f..2e2b86471b0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -43,7 +43,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.time.Duration; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -80,7 +79,6 @@ public class OMDBCheckpointServlet extends DBCheckpointServlet { private static final Logger LOG = LoggerFactory.getLogger(OMDBCheckpointServlet.class); private static final long serialVersionUID = 1L; - private static final String DURATION_TO_WAIT_FOR_DIRECTORY = "PT10S"; @Override public void init() throws ServletException { @@ -139,7 +137,7 @@ private void getFilesForArchive(DBCheckpoint checkpoint, Map copyFiles, Map hardLinkFiles, boolean includeSnapshotData) - throws IOException, InterruptedException { + throws IOException { // Get the active fs files. Path dir = checkpoint.getCheckpointLocation(); @@ -163,7 +161,7 @@ private void getFilesForArchive(DBCheckpoint checkpoint, * @return Set of expected snapshot dirs. */ private Set waitForSnapshotDirs(DBCheckpoint checkpoint) - throws IOException, InterruptedException { + throws IOException { OzoneConfiguration conf = getConf(); @@ -206,6 +204,7 @@ private void processDir(Path dir, Map copyFiles, String parent = f.getParent(); if (parent != null && parent.endsWith(OM_SNAPSHOT_CHECKPOINT_DIR) && !snapshotPaths.contains(file)) { + LOG.debug("Skipping unneeded file: " + file); continue; } processDir(file, copyFiles, hardLinkFiles, snapshotPaths); From f77b4d6a91bde3668ed8167ae98325590daea054 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 21 Mar 2023 14:49:58 -0700 Subject: [PATCH 107/120] checkstyle --- .../hdds/utils/DBCheckpointServlet.java | 5 ++--- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 20 ++++++++----------- .../snapshot/TestOzoneManagerSnapshotAcl.java | 20 +++++++------------ 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 618c752f314..21b7c7cfde6 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -190,14 +190,13 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { } /** - * Write checkpoint to the stream + * Write checkpoint to the stream. * * @param checkpoint The checkpoint to be written. - * @param ignoredRequest The httpRequest which generated this checkpoint + * @param ignoredRequest The httpRequest which generated this checkpoint. * (Parameter is ignored in this class but used in child classes). * @param destination The stream to write to. */ - @Override public void writeDbDataToStream(DBCheckpoint checkpoint, HttpServletRequest ignoredRequest, OutputStream destination) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 8a4f07194b8..c8a80811532 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -65,12 +65,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithData.createKey; import static org.junit.Assert.assertEquals; @@ -180,7 +178,7 @@ public void testInstallSnapshot() throws Exception { // Do some transactions so that the log index increases List keys = writeKeysToIncreaseLogIndex(leaderRatisServer, 200); - SnapshotInfo snapshotInfo = createOzoneSnapshot(leaderOM, keys); + SnapshotInfo snapshotInfo = createOzoneSnapshot(leaderOM); // Get the latest db checkpoint from the leader OM. TransactionInfo transactionInfo = @@ -258,7 +256,7 @@ public void testInstallSnapshot() throws Exception { .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(".snapshot/snap1/" + keys.get(0)).build(); - OmKeyInfo omKeyInfo = null; + OmKeyInfo omKeyInfo; omKeyInfo = followerOM.lookupKey(omKeyArgs); Assertions.assertNotNull(omKeyInfo); Assertions.assertEquals(omKeyInfo.getKeyName(), omKeyArgs.getKeyName()); @@ -272,8 +270,6 @@ public void testInstallSnapshot() throws Exception { Path leaderActiveDir = Paths.get(leaderMetaDir.toString(), OM_DB_NAME); Path leaderSnapshotDir = Paths.get(getSnapshotPath(leaderOM.getConfiguration(), snapshotInfo)); - // Check snapshot hard links - // Get the list of hardlinks from the leader. Then confirm those links // are on the follower int hardLinkCount = 0; @@ -293,16 +289,16 @@ public void testInstallSnapshot() throws Exception { .equals(OmSnapshotManager.getINode(leaderSnapshotSST))) { Path followerSnapshotSST = Paths.get(followerSnapshotDir.toString(), fileName); Path followerActiveSST = Paths.get(followerActiveDir.toString(), fileName); - assertEquals("Snapshot sst file is supposed to be a hard link", - OmSnapshotManager.getINode(followerActiveSST), - OmSnapshotManager.getINode(followerSnapshotSST)); + Assertions.assertEquals( + OmSnapshotManager.getINode(followerActiveSST), + OmSnapshotManager.getINode(followerSnapshotSST), + "Snapshot sst file is supposed to be a hard link"); hardLinkCount++; } - } } } - assertTrue("No hard links were found", hardLinkCount > 0); + Assertions.assertTrue(hardLinkCount > 0, "No hard links were found"); } @Ignore("Enable this unit test after RATIS-1481 used") @@ -613,7 +609,7 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { Assert.assertTrue(logCapture.getOutput().contains(msg)); } - private SnapshotInfo createOzoneSnapshot(OzoneManager leaderOM, List keys) + private SnapshotInfo createOzoneSnapshot(OzoneManager leaderOM) throws IOException { objectStore.createSnapshot(volumeName, bucketName, "snap1"); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java index b2eb4bf72a4..e2ed0d0940d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.IOException; import java.util.UUID; -import java.util.concurrent.TimeoutException; import java.util.stream.Stream; import org.apache.hadoop.hdds.utils.IOUtils; @@ -50,7 +49,6 @@ import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -67,9 +65,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConsts.ADMIN; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -97,7 +92,6 @@ public class TestOzoneManagerSnapshotAcl { OzoneObj.ResourceType.KEY; private static MiniOzoneCluster cluster; private static ObjectStore objectStore; - private static File metaDir; private static OzoneManager ozoneManager; private static OzoneClient client; private String volumeName; @@ -140,7 +134,7 @@ public static void init() throws Exception { // stop the deletion services so that keys can still be read keyManager.stop(); - metaDir = OMStorage.getOmDbDir(ozoneManagerConf); + OMStorage.getOmDbDir(ozoneManagerConf); } @AfterEach @@ -193,7 +187,7 @@ public void testLookupKeyWithNotAllowedUser(BucketLayout bucketLayout) @ParameterizedTest @EnumSource(BucketLayout.class) public void testGeyKeyInfoWithAllowedUser(BucketLayout bucketLayout) - throws IOException, InterruptedException, TimeoutException { + throws IOException { // GIVEN setup(bucketLayout); final OmKeyArgs snapshotKeyArgs = getOmKeyArgs(true); @@ -207,7 +201,7 @@ public void testGeyKeyInfoWithAllowedUser(BucketLayout bucketLayout) @ParameterizedTest @EnumSource(BucketLayout.class) public void testGeyKeyInfoWithNotAllowedUser(BucketLayout bucketLayout) - throws IOException, InterruptedException, TimeoutException { + throws IOException { // GIVEN setup(bucketLayout); final OmKeyArgs snapshotKeyOmKeyArgs = getOmKeyArgs(true); @@ -233,7 +227,7 @@ public void testGeyKeyInfoWithNotAllowedUser(BucketLayout bucketLayout) @MethodSource("getListStatusArguments") public void testListStatusWithAllowedUser(BucketLayout bucketLayout, boolean recursive, boolean allowPartialPrefixes) - throws IOException, InterruptedException, TimeoutException { + throws IOException { // GIVEN setup(bucketLayout); final OmKeyArgs snapshotKeyArgs = getOmKeyArgs(true); @@ -250,7 +244,7 @@ public void testListStatusWithAllowedUser(BucketLayout bucketLayout, @MethodSource("getListStatusArguments") public void testListStatusWithNotAllowedUser(BucketLayout bucketLayout, boolean recursive, boolean allowPartialPrefixes) - throws IOException, InterruptedException, TimeoutException { + throws IOException { // GIVEN setup(bucketLayout); final OmKeyArgs snapshotKeyArgs = getOmKeyArgs(true); @@ -394,7 +388,7 @@ public void testGetAclWithNotAllowedUser(BucketLayout bucketLayout) } private void setup(BucketLayout bucketLayout) - throws IOException, InterruptedException, TimeoutException { + throws IOException { UserGroupInformation.setLoginUser(UGI1); createVolume(); @@ -470,7 +464,7 @@ private void createKey(OzoneBucket bucket) } private void createSnapshot() - throws IOException, InterruptedException, TimeoutException { + throws IOException { final String snapshotPrefix = "snapshot-"; final String snapshotName = snapshotPrefix + RandomStringUtils.randomNumeric(5); From 051b97bf3eecaec31622161ee386002361e19c99 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 21 Mar 2023 14:56:49 -0700 Subject: [PATCH 108/120] checkstyle --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 14 ++++++++------ .../hadoop/ozone/om/OmMetadataManagerImpl.java | 3 ++- .../hadoop/ozone/om/SstFilteringService.java | 5 +++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index c8a80811532..fb6600d5a62 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -71,7 +71,6 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithData.createKey; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -284,11 +283,14 @@ public void testInstallSnapshot() throws Exception { if (!leaderActiveSST.toFile().exists()) { continue; } - // If it is a hard link on the leader, it should be a hard link on the follower + // If it is a hard link on the leader, it should be a hard + // link on the follower if (OmSnapshotManager.getINode(leaderActiveSST) .equals(OmSnapshotManager.getINode(leaderSnapshotSST))) { - Path followerSnapshotSST = Paths.get(followerSnapshotDir.toString(), fileName); - Path followerActiveSST = Paths.get(followerActiveDir.toString(), fileName); + Path followerSnapshotSST = + Paths.get(followerSnapshotDir.toString(), fileName); + Path followerActiveSST = + Paths.get(followerActiveDir.toString(), fileName); Assertions.assertEquals( OmSnapshotManager.getINode(followerActiveSST), OmSnapshotManager.getINode(followerSnapshotSST), @@ -617,8 +619,8 @@ private SnapshotInfo createOzoneSnapshot(OzoneManager leaderOM) bucketName, "snap1"); SnapshotInfo snapshotInfo = leaderOM.getMetadataManager() - .getSnapshotInfoTable() - .get(tableKey); + .getSnapshotInfoTable() + .get(tableKey); // Allow the snapshot to be written to disk String fileName = getSnapshotPath(leaderOM.getConfiguration(), snapshotInfo); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 93b5f2f0fa7..0ade8ea7aca 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -340,7 +340,8 @@ public static OmMetadataManagerImpl createCheckpointMetadataManager( return new OmMetadataManagerImpl(conf, dir, name); // NPE check forced by findbugs NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE } catch (NullPointerException e) { - throw new IOException("Invalid database: " + checkpoint.getCheckpointLocation()); + throw new IOException("Invalid database: " + + checkpoint.getCheckpointLocation()); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index aae14d6db20..cdc473646b7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -148,8 +148,9 @@ public BackgroundTaskResult call() throws Exception { String snapshotCheckpointDir = omMetadataDir + OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR; try (RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl - .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), - dbName, true, Optional.of(Boolean.TRUE))) { + .loadDB(ozoneManager.getConfiguration(), + new File(snapshotCheckpointDir), + dbName, true, Optional.of(Boolean.TRUE))) { RocksDatabase db = rdbStore.getDb(); db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction); } From dbcaae91faff21538c865f7d5620769ae4aeced9 Mon Sep 17 00:00:00 2001 From: GeorgeJahad Date: Sun, 26 Mar 2023 23:05:45 -0700 Subject: [PATCH 109/120] Apply suggestions from code review Co-authored-by: Hemant Kumar Co-authored-by: Siyao Meng <50227127+smengcl@users.noreply.github.com> --- .../org/apache/hadoop/ozone/OzoneConsts.java | 2 +- .../apache/hadoop/hdds/utils/db/RDBStore.java | 2 +- .../ozone/om/OMDBCheckpointServlet.java | 2 ++ .../ozone/om/OmMetadataManagerImpl.java | 20 ++++++++++--------- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 084836ac90d..cf807e4145f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -572,7 +572,7 @@ private OzoneConsts() { + OM_KEY_PREFIX + "diffState"; // Name of the SST file backup directory placed under diff dir. - public static final String OM_COMPACTION_BACKUP_DIR = "compaction-sst-backup"; + public static final String OM_COMPACTION_SST_BACKUP_DIR = "compaction-sst-backup"; // Name of the compaction log directory placed under diff dir. public static final String OM_COMPACTION_LOG_DIR = "compaction-log"; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 2e0aab81145..0cad4df2d04 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -130,7 +130,7 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, //create checkpoints directory if not exists. checkpointsParentDir = - Paths.get(dbLocation.getParent(), OM_CHECKPOINT_DIR).toString(); + dbLocation.getParent() + OM_KEY_PREFIX + OM_CHECKPOINT_DIR; File checkpointsDir = new File(checkpointsParentDir); if (!checkpointsDir.exists()) { boolean success = checkpointsDir.mkdir(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 2e2b86471b0..e76d14d85f8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -231,6 +231,8 @@ private void processFile(Path file, Map copyFiles, private boolean includeSnapshotData(HttpServletRequest request) { String includeParam = request.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA); + return Boolean.parseBoolean(includeParam); + request.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA); if (StringUtils.isNotEmpty(includeParam)) { return Boolean.parseBoolean(includeParam); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 0ade8ea7aca..d0f0798d9b9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -333,16 +333,18 @@ protected OmMetadataManagerImpl() { public static OmMetadataManagerImpl createCheckpointMetadataManager( OzoneConfiguration conf, DBCheckpoint checkpoint) throws IOException { Path path = checkpoint.getCheckpointLocation(); - try { - Path parent = path.getParent(); - File dir = parent.toFile(); - String name = path.getFileName().toString(); - return new OmMetadataManagerImpl(conf, dir, name); - // NPE check forced by findbugs NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE - } catch (NullPointerException e) { - throw new IOException("Invalid database: " + - checkpoint.getCheckpointLocation()); + Path parent = path.getParent(); + if (parent == null) { + throw new IllegalStateException("DB checkpoint parent path should not " + + "have been null. Checkpoint path is " + path); + } + File dir = parent.toFile(); + Path name = path.getFileName(); + if (name == null) { + throw new IllegalStateException("DB checkpoint dir name should not " + + "have been null. Checkpoint path is " + path); } + return new OmMetadataManagerImpl(conf, dir, name.toString()); } /** From bb6e09f97537d89ed35cad08efa5598330b6fc78 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 27 Mar 2023 09:00:06 -0700 Subject: [PATCH 110/120] more review changes --- .../main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java | 4 ++-- .../apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java | 4 ++-- .../org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 5 ----- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 0cad4df2d04..54c3fd26859 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; -import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_BACKUP_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_SST_BACKUP_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_LOG_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; @@ -101,7 +101,7 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, if (enableCompactionLog) { rocksDBCheckpointDiffer = new RocksDBCheckpointDiffer( dbLocation.getParent() + OM_KEY_PREFIX + OM_SNAPSHOT_DIFF_DIR, - OM_COMPACTION_BACKUP_DIR, OM_COMPACTION_LOG_DIR, + OM_COMPACTION_SST_BACKUP_DIR, OM_COMPACTION_LOG_DIR, dbLocation.toString(), maxTimeAllowedForSnapshotInDag, compactionDagDaemonInterval); rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 444deb5bb0a..ecaba6e7306 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -65,7 +65,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_BACKUP_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_COMPACTION_SST_BACKUP_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; @@ -437,7 +437,7 @@ private void prepSnapshotData() throws Exception { // Simulate links from the compaction dir. compactionDirPath = Paths.get(metaDir.toString(), - OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_BACKUP_DIR); + OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_SST_BACKUP_DIR); Path fabricatedLink2 = Paths.get(compactionDirPath.toString(), "fabricatedFile"); Files.createLink(fabricatedLink2, fabricatedFile); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index e76d14d85f8..9adcc449f2c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -232,11 +232,6 @@ private boolean includeSnapshotData(HttpServletRequest request) { String includeParam = request.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA); return Boolean.parseBoolean(includeParam); - request.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA); - if (StringUtils.isNotEmpty(includeParam)) { - return Boolean.parseBoolean(includeParam); - } - return false; } private void writeFilesToArchive(Map copyFiles, From b7ede37839e76494cb0491390f32225a72b8fc8d Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 27 Mar 2023 20:34:21 -0700 Subject: [PATCH 111/120] refactoring based on review comments --- .../apache/hadoop/hdds/utils/db/RDBStore.java | 4 +- .../ozone/om/TestOMDbCheckpointServlet.java | 2 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 9 +- .../ozone/om/OMDBCheckpointServlet.java | 8 +- .../hadoop/ozone/om/OmSnapshotManager.java | 106 +------------- .../apache/hadoop/ozone/om/OzoneManager.java | 3 +- .../ozone/om/snapshot/OmSnapshotUtils.java | 135 ++++++++++++++++++ .../ozone/om/TestOmSnapshotManager.java | 7 +- 8 files changed, 154 insertions(+), 120 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index e44f4fd79cd..d6f7cd589b2 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -143,7 +143,7 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, } } - //create snapshot directory if does not exist. + //create snapshot checkpoint directory if does not exist. snapshotsParentDir = Paths.get(dbLocation.getParent(), OM_SNAPSHOT_CHECKPOINT_DIR).toString(); File snapshotsDir = new File(snapshotsParentDir); @@ -151,7 +151,7 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, boolean success = snapshotsDir.mkdirs(); if (!success) { throw new IOException( - "Unable to create RocksDB snapshot directory: " + + "Unable to create RocksDB snapshot checkpoint directory: " + snapshotsParentDir); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index af6a3371071..316130c2e2f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -85,7 +85,7 @@ import org.mockito.Matchers; import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE; -import static org.apache.hadoop.ozone.om.OmSnapshotManager.truncateFileName; +import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.truncateFileName; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index fb6600d5a62..11e7455827a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; +import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils; import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.server.protocol.TermIndex; import org.assertj.core.api.Fail; @@ -285,15 +286,15 @@ public void testInstallSnapshot() throws Exception { } // If it is a hard link on the leader, it should be a hard // link on the follower - if (OmSnapshotManager.getINode(leaderActiveSST) - .equals(OmSnapshotManager.getINode(leaderSnapshotSST))) { + if (OmSnapshotUtils.getINode(leaderActiveSST) + .equals(OmSnapshotUtils.getINode(leaderSnapshotSST))) { Path followerSnapshotSST = Paths.get(followerSnapshotDir.toString(), fileName); Path followerActiveSST = Paths.get(followerActiveDir.toString(), fileName); Assertions.assertEquals( - OmSnapshotManager.getINode(followerActiveSST), - OmSnapshotManager.getINode(followerSnapshotSST), + OmSnapshotUtils.getINode(followerActiveSST), + OmSnapshotUtils.getINode(followerSnapshotSST), "Snapshot sst file is supposed to be a hard link"); hardLinkCount++; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 9adcc449f2c..fb4995adc38 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -20,7 +20,6 @@ import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.recon.ReconConfig; import org.apache.hadoop.hdds.server.ServerUtils; @@ -31,6 +30,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,9 +57,9 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA; -import static org.apache.hadoop.ozone.om.OmSnapshotManager.createHardLinkList; +import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.createHardLinkList; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; -import static org.apache.hadoop.ozone.om.OmSnapshotManager.truncateFileName; +import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.truncateFileName; /** * Provides the current checkpoint Snapshot of the OM DB. (tar.gz) @@ -218,7 +218,7 @@ private void processDir(Path dir, Map copyFiles, private void processFile(Path file, Map copyFiles, Map hardLinkFiles) throws IOException { // Get the inode. - Object key = OmSnapshotManager.getINode(file); + Object key = OmSnapshotUtils.getINode(file); // If we already have the inode, store as hard link. if (copyFiles.containsKey(key)) { hardLinkFiles.put(file, copyFiles.get(key)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 4d7f59f4504..196ae2f8a4b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -63,14 +62,7 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Map; -import java.util.stream.Stream; - -import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; + import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.hadoop.hdds.utils.db.DBStoreBuilder.DEFAULT_COLUMN_FAMILY_NAME; @@ -374,12 +366,6 @@ private static void deleteKeysInSnapshotScopeFromDTableInternal( // one by one. } - @VisibleForTesting - static Object getINode(Path file) throws IOException { - return Files.readAttributes( - file, BasicFileAttributes.class).fileKey(); - } - // Get OmSnapshot if the keyname has ".snapshot" key indicator public IOmMetadataReader checkForSnapshot(String volumeName, String bucketName, String keyname) @@ -497,96 +483,6 @@ private void verifySnapshotInfoForSnapDiff(final SnapshotInfo fromSnapshot, } } - /** - * Create file of links to add to tarball. - * Format of entries are either: - * dir1/fileTo fileFrom - * for files in active db or: - * dir1/fileTo dir2/fileFrom - * for files in another directory, (either another snapshot dir or - * sst compaction backup directory) - * @param truncateLength - Length of initial path to trim in file path. - * @param hardLinkFiles - Map of link->file paths. - * @return Path to the file of links created. - */ - static Path createHardLinkList(int truncateLength, - Map hardLinkFiles) - throws IOException { - Path data = Files.createTempFile("data", "txt"); - StringBuilder sb = new StringBuilder(); - for (Map.Entry entry : hardLinkFiles.entrySet()) { - String fixedFile = truncateFileName(truncateLength, entry.getValue()); - // If this file is from the active db, strip the path. - if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { - Path f = Paths.get(fixedFile).getFileName(); - if (f != null) { - fixedFile = f.toString(); - } - } - sb.append(truncateFileName(truncateLength, entry.getKey())) - .append("\t") - .append(fixedFile) - .append("\n"); - } - Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); - return data; - } - - /** - * Get the filename without the introductory metadata directory. - * - * @param truncateLength Length to remove. - * @param file File to remove prefix from. - * @return Truncated string. - */ - static String truncateFileName(int truncateLength, Path file) { - return file.toString().substring(truncateLength); - } - - /** - * Create hard links listed in OM_HARDLINK_FILE. - * - * @param dbPath Path to db to have links created. - */ - static void createHardLinks(Path dbPath) throws IOException { - File hardLinkFile = new File(dbPath.toString(), - OM_HARDLINK_FILE); - if (hardLinkFile.exists()) { - // Read file. - try (Stream s = Files.lines(hardLinkFile.toPath())) { - List lines = s.collect(Collectors.toList()); - - // Create a link for each line. - for (String l : lines) { - String from = l.split("\t")[1]; - String to = l.split("\t")[0]; - Path fullFromPath = getFullPath(dbPath, from); - Path fullToPath = getFullPath(dbPath, to); - Files.createLink(fullToPath, fullFromPath); - } - if (!hardLinkFile.delete()) { - throw new IOException( - "Failed to delete: " + hardLinkFile); - } - } - } - } - // Prepend the full path to the hard link entry entry. - private static Path getFullPath(Path dbPath, String fileName) - throws IOException { - File file = new File(fileName); - // If there is no directory then this file belongs in the db. - if (file.getName().equals(fileName)) { - return Paths.get(dbPath.toString(), fileName); - } - // Else this file belong in a directory parallel to the db. - Path parent = dbPath.getParent(); - if (parent == null) { - throw new IOException("Invalid database " + dbPath); - } - return Paths.get(parent.toString(), fileName); - } - private int getIndexFromToken(final String token) throws IOException { if (isBlank(token)) { return 0; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 8a72ede488b..68d1191ed8d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -90,6 +90,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.service.OMRangerBGSyncService; +import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.util.OzoneNetUtils; @@ -3821,7 +3822,7 @@ private void moveOmSnapshotData(Path dbPath, Path dbSnapshotsDir) OM_SNAPSHOT_DIR); if (incomingSnapshotsDir.toFile().exists()) { Files.move(incomingSnapshotsDir, dbSnapshotsDir); - OmSnapshotManager.createHardLinks(dbPath); + OmSnapshotUtils.createHardLinks(dbPath); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java new file mode 100644 index 00000000000..266c4c38361 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.ozone.om.OmSnapshotManager; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; + +public class OmSnapshotUtils { + /** + * Get the filename without the introductory metadata directory. + * + * @param truncateLength Length to remove. + * @param file File to remove prefix from. + * @return Truncated string. + */ + public static String truncateFileName(int truncateLength, Path file) { + return file.toString().substring(truncateLength); + } + + @VisibleForTesting + public static Object getINode(Path file) throws IOException { + return Files.readAttributes( + file, BasicFileAttributes.class).fileKey(); + } + + /** + * Create file of links to add to tarball. + * Format of entries are either: + * dir1/fileTo fileFrom + * for files in active db or: + * dir1/fileTo dir2/fileFrom + * for files in another directory, (either another snapshot dir or + * sst compaction backup directory) + * @param truncateLength - Length of initial path to trim in file path. + * @param hardLinkFiles - Map of link->file paths. + * @return Path to the file of links created. + */ + public static Path createHardLinkList(int truncateLength, + Map hardLinkFiles) + throws IOException { + Path data = Files.createTempFile("data", "txt"); + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : hardLinkFiles.entrySet()) { + String fixedFile = truncateFileName(truncateLength, entry.getValue()); + // If this file is from the active db, strip the path. + if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) { + Path f = Paths.get(fixedFile).getFileName(); + if (f != null) { + fixedFile = f.toString(); + } + } + sb.append(truncateFileName(truncateLength, entry.getKey())) + .append("\t") + .append(fixedFile) + .append("\n"); + } + Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); + return data; + } + + /** + * Create hard links listed in OM_HARDLINK_FILE. + * + * @param dbPath Path to db to have links created. + */ + public static void createHardLinks(Path dbPath) throws IOException { + File hardLinkFile = new File(dbPath.toString(), + OmSnapshotManager.OM_HARDLINK_FILE); + if (hardLinkFile.exists()) { + // Read file. + try (Stream s = Files.lines(hardLinkFile.toPath())) { + List lines = s.collect(Collectors.toList()); + + // Create a link for each line. + for (String l : lines) { + String from = l.split("\t")[1]; + String to = l.split("\t")[0]; + Path fullFromPath = getFullPath(dbPath, from); + Path fullToPath = getFullPath(dbPath, to); + Files.createLink(fullToPath, fullFromPath); + } + if (!hardLinkFile.delete()) { + throw new IOException( + "Failed to delete: " + hardLinkFile); + } + } + } + } + + // Prepend the full path to the hard link entry entry. + private static Path getFullPath(Path dbPath, String fileName) + throws IOException { + File file = new File(fileName); + // If there is no directory then this file belongs in the db. + if (file.getName().equals(fileName)) { + return Paths.get(dbPath.toString(), fileName); + } + // Else this file belong in a directory parallel to the db. + Path parent = dbPath.getParent(); + if (parent == null) { + throw new IOException("Invalid database " + dbPath); + } + return Paths.get(parent.toString(), fileName); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index b4ff0a50874..b31596b233e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils; import org.apache.ozone.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; @@ -48,7 +49,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE; -import static org.apache.hadoop.ozone.om.OmSnapshotManager.getINode; +import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.getINode; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; @@ -160,12 +161,12 @@ public void testHardLinkCreation() throws IOException { // Create link list. Path hardLinkList = - OmSnapshotManager.createHardLinkList( + OmSnapshotUtils.createHardLinkList( testDir.toString().length() + 1, hardLinkFiles); Files.move(hardLinkList, Paths.get(dbDir.toString(), OM_HARDLINK_FILE)); // Create links from list. - OmSnapshotManager.createHardLinks(dbDir.toPath()); + OmSnapshotUtils.createHardLinks(dbDir.toPath()); // Confirm expected links. for (Map.Entry entry : hardLinkFiles.entrySet()) { From 2bb1fa57e0e0a10a5126140b5520f6e5e2d43c14 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 27 Mar 2023 20:36:13 -0700 Subject: [PATCH 112/120] fix alignment --- .../ozone/om/snapshot/OmSnapshotUtils.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java index 266c4c38361..9d28e7eb8ac 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java @@ -40,7 +40,7 @@ public class OmSnapshotUtils { * Get the filename without the introductory metadata directory. * * @param truncateLength Length to remove. - * @param file File to remove prefix from. + * @param file File to remove prefix from. * @return Truncated string. */ public static String truncateFileName(int truncateLength, Path file) { @@ -49,20 +49,20 @@ public static String truncateFileName(int truncateLength, Path file) { @VisibleForTesting public static Object getINode(Path file) throws IOException { - return Files.readAttributes( - file, BasicFileAttributes.class).fileKey(); + return Files.readAttributes(file, BasicFileAttributes.class).fileKey(); } /** * Create file of links to add to tarball. * Format of entries are either: * dir1/fileTo fileFrom - * for files in active db or: + * for files in active db or: * dir1/fileTo dir2/fileFrom - * for files in another directory, (either another snapshot dir or - * sst compaction backup directory) + * for files in another directory, (either another snapshot dir or + * sst compaction backup directory) + * * @param truncateLength - Length of initial path to trim in file path. - * @param hardLinkFiles - Map of link->file paths. + * @param hardLinkFiles - Map of link->file paths. * @return Path to the file of links created. */ public static Path createHardLinkList(int truncateLength, @@ -79,10 +79,8 @@ public static Path createHardLinkList(int truncateLength, fixedFile = f.toString(); } } - sb.append(truncateFileName(truncateLength, entry.getKey())) - .append("\t") - .append(fixedFile) - .append("\n"); + sb.append(truncateFileName(truncateLength, entry.getKey())).append("\t") + .append(fixedFile).append("\n"); } Files.write(data, sb.toString().getBytes(StandardCharsets.UTF_8)); return data; @@ -94,8 +92,8 @@ public static Path createHardLinkList(int truncateLength, * @param dbPath Path to db to have links created. */ public static void createHardLinks(Path dbPath) throws IOException { - File hardLinkFile = new File(dbPath.toString(), - OmSnapshotManager.OM_HARDLINK_FILE); + File hardLinkFile = + new File(dbPath.toString(), OmSnapshotManager.OM_HARDLINK_FILE); if (hardLinkFile.exists()) { // Read file. try (Stream s = Files.lines(hardLinkFile.toPath())) { @@ -110,8 +108,7 @@ public static void createHardLinks(Path dbPath) throws IOException { Files.createLink(fullToPath, fullFromPath); } if (!hardLinkFile.delete()) { - throw new IOException( - "Failed to delete: " + hardLinkFile); + throw new IOException("Failed to delete: " + hardLinkFile); } } } From 8d33bb40e023c7a10a5ca268317ce67078ea6387 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 27 Mar 2023 20:48:04 -0700 Subject: [PATCH 113/120] moved waitForCheckpointDirectoryExist into utils class --- .../hdds/utils/db/RDBCheckpointManager.java | 38 +---------- .../hdds/utils/db/RDBCheckpointUtils.java | 63 +++++++++++++++++++ .../hadoop/ozone/om/TestOMRatisSnapshots.java | 4 +- .../snapshot/TestOzoneManagerSnapshotAcl.java | 4 +- .../ozone/om/OMDBCheckpointServlet.java | 4 +- .../ozone/om/OmMetadataManagerImpl.java | 4 +- 6 files changed, 72 insertions(+), 45 deletions(-) create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java index 1651553af1b..d18c62f58c6 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hdds.utils.db; import java.io.Closeable; -import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -28,11 +27,6 @@ import java.time.Instant; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.utils.db.RocksDatabase.RocksCheckpoint; -import org.awaitility.core.ConditionTimeoutException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.awaitility.Awaitility.with; /** * RocksDB Checkpoint Manager, used to create and cleanup checkpoints. @@ -41,12 +35,7 @@ public class RDBCheckpointManager implements Closeable { private final RocksDatabase db; private final RocksCheckpoint checkpoint; public static final String RDB_CHECKPOINT_DIR_PREFIX = "checkpoint_"; - private static final Logger LOG = - LoggerFactory.getLogger(RDBCheckpointManager.class); private final String checkpointNamePrefix; - private static final Duration POLL_DELAY_DURATION = Duration.ZERO; - private static final Duration POLL_INTERVAL_DURATION = Duration.ofMillis(100); - private static final Duration POLL_MAX_DURATION = Duration.ofSeconds(5); /** * Create a checkpoint manager with a prefix to be added to the @@ -96,7 +85,7 @@ public RocksDBCheckpoint createCheckpoint(String parentDir, String name) { LOG.info("Created checkpoint in rocksDB at {} in {} milliseconds", checkpointPath, duration); - waitForCheckpointDirectoryExist(checkpointPath.toFile()); + RDBCheckpointUtils.waitForCheckpointDirectoryExist(checkpointPath.toFile()); return new RocksDBCheckpoint( checkpointPath, @@ -109,31 +98,6 @@ public RocksDBCheckpoint createCheckpoint(String parentDir, String name) { return null; } - /** - * Wait for checkpoint directory to be created for 5 secs with 100 millis - * poll interval. - */ - public static boolean waitForCheckpointDirectoryExist(File file) - throws IOException { - Instant start = Instant.now(); - try { - with().atMost(POLL_MAX_DURATION) - .pollDelay(POLL_DELAY_DURATION) - .pollInterval(POLL_INTERVAL_DURATION) - .await() - .until(file::exists); - LOG.info("Waited for {} milliseconds for checkpoint directory {}" + - " availability.", - Duration.between(start, Instant.now()).toMillis(), - file.getAbsoluteFile()); - return true; - } catch (ConditionTimeoutException exception) { - LOG.info("Checkpoint directory: {} didn't get created in 5 secs.", - file.getAbsolutePath()); - return false; - } - } - /** * Create RocksDB snapshot by saving a checkpoint to a directory. * diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java new file mode 100644 index 00000000000..4f4c2f471ac --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.utils.db; + +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; + +import static org.awaitility.Awaitility.with; + +public class RDBCheckpointUtils { + static final Logger LOG = + LoggerFactory.getLogger(RDBCheckpointUtils.class); + private static final Duration POLL_DELAY_DURATION = Duration.ZERO; + private static final Duration POLL_INTERVAL_DURATION = Duration.ofMillis(100); + private static final Duration POLL_MAX_DURATION = Duration.ofSeconds(5); + + /** + * Wait for checkpoint directory to be created for 5 secs with 100 millis + * poll interval. + */ + public static boolean waitForCheckpointDirectoryExist(File file) + throws IOException { + Instant start = Instant.now(); + try { + with().atMost(POLL_MAX_DURATION) + .pollDelay(POLL_DELAY_DURATION) + .pollInterval(POLL_INTERVAL_DURATION) + .await() + .until(file::exists); + LOG.info("Waited for {} milliseconds for checkpoint directory {}" + + " availability.", + Duration.between(start, Instant.now()).toMillis(), + file.getAbsoluteFile()); + return true; + } catch (ConditionTimeoutException exception) { + LOG.info("Checkpoint directory: {} didn't get created in 5 secs.", + file.getAbsolutePath()); + return false; + } + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 11e7455827a..31ea777b7d7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; -import org.apache.hadoop.hdds.utils.db.RDBCheckpointManager; +import org.apache.hadoop.hdds.utils.db.RDBCheckpointUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.client.BucketArgs; @@ -626,7 +626,7 @@ private SnapshotInfo createOzoneSnapshot(OzoneManager leaderOM) String fileName = getSnapshotPath(leaderOM.getConfiguration(), snapshotInfo); File snapshotDir = new File(fileName); - if (!RDBCheckpointManager + if (!RDBCheckpointUtils .waitForCheckpointDirectoryExist(snapshotDir)) { throw new IOException("snapshot directory doesn't exist"); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java index e2ed0d0940d..0c12a7fc899 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java @@ -28,7 +28,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; -import org.apache.hadoop.hdds.utils.db.RDBCheckpointManager; +import org.apache.hadoop.hdds.utils.db.RDBCheckpointUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.BucketArgs; @@ -479,7 +479,7 @@ private void createSnapshot() String fileName = getSnapshotPath(ozoneManager.getConfiguration(), snapshotInfo); File snapshotDir = new File(fileName); - if (!RDBCheckpointManager + if (!RDBCheckpointUtils .waitForCheckpointDirectoryExist(snapshotDir)) { throw new IOException("snapshot directory doesn't exist"); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index fb4995adc38..023f2cb3e1e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.DBCheckpointServlet; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; -import org.apache.hadoop.hdds.utils.db.RDBCheckpointManager; +import org.apache.hadoop.hdds.utils.db.RDBCheckpointUtils; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneConsts; @@ -187,7 +187,7 @@ private Set waitForSnapshotDirs(DBCheckpoint checkpoint) } private void waitForDirToExist(Path dir) throws IOException { - if (!RDBCheckpointManager.waitForCheckpointDirectoryExist(dir.toFile())) { + if (!RDBCheckpointUtils.waitForCheckpointDirectoryExist(dir.toFile())) { throw new IOException("snapshot dir doesn't exist: " + dir); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index fdf766a390a..a5fbd60c1be 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -42,9 +42,9 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; +import org.apache.hadoop.hdds.utils.db.RDBCheckpointUtils; import org.apache.hadoop.hdds.utils.db.RocksDBConfiguration; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.RDBCheckpointManager; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.db.TypedTable; @@ -378,7 +378,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) // it is most likely DB entries will get flushed in this wait time. if (isSnapshotInCache) { File checkpoint = Paths.get(metaDir.toPath().toString(), dbName).toFile(); - RDBCheckpointManager.waitForCheckpointDirectoryExist(checkpoint); + RDBCheckpointUtils.waitForCheckpointDirectoryExist(checkpoint); } setStore(loadDB(conf, metaDir, dbName, true, java.util.Optional.of(Boolean.TRUE))); From 0212d2fb68887f3eb4849ddaa1e77357b9471ecd Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 27 Mar 2023 21:07:46 -0700 Subject: [PATCH 114/120] checkstyle --- .../hadoop/hdds/utils/db/RDBCheckpointManager.java | 7 ++++++- .../hadoop/hdds/utils/db/RDBCheckpointUtils.java | 7 +++++++ .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 1 - .../hadoop/ozone/om/snapshot/OmSnapshotUtils.java | 11 +++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java index d18c62f58c6..e3b353164d6 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java @@ -27,6 +27,8 @@ import java.time.Instant; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.utils.db.RocksDatabase.RocksCheckpoint; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * RocksDB Checkpoint Manager, used to create and cleanup checkpoints. @@ -35,6 +37,8 @@ public class RDBCheckpointManager implements Closeable { private final RocksDatabase db; private final RocksCheckpoint checkpoint; public static final String RDB_CHECKPOINT_DIR_PREFIX = "checkpoint_"; + private static final Logger LOG = + LoggerFactory.getLogger(RDBCheckpointManager.class); private final String checkpointNamePrefix; /** @@ -85,7 +89,8 @@ public RocksDBCheckpoint createCheckpoint(String parentDir, String name) { LOG.info("Created checkpoint in rocksDB at {} in {} milliseconds", checkpointPath, duration); - RDBCheckpointUtils.waitForCheckpointDirectoryExist(checkpointPath.toFile()); + RDBCheckpointUtils.waitForCheckpointDirectoryExist( + checkpointPath.toFile()); return new RocksDBCheckpoint( checkpointPath, diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java index 4f4c2f471ac..3e6b7635671 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java @@ -29,6 +29,9 @@ import static org.awaitility.Awaitility.with; +/** + * RocksDB Checkpoint Utilities. + */ public class RDBCheckpointUtils { static final Logger LOG = LoggerFactory.getLogger(RDBCheckpointUtils.class); @@ -36,9 +39,13 @@ public class RDBCheckpointUtils { private static final Duration POLL_INTERVAL_DURATION = Duration.ofMillis(100); private static final Duration POLL_MAX_DURATION = Duration.ofSeconds(5); + private RDBCheckpointUtils(){} + /** * Wait for checkpoint directory to be created for 5 secs with 100 millis * poll interval. + * @param file Checkpoint directory. + * @return true if found. */ public static boolean waitForCheckpointDirectoryExist(File file) throws IOException { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index d6f7cd589b2..5d2c2c85f6c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -55,7 +55,6 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DIR; import static org.apache.hadoop.ozone.OzoneConsts.DB_COMPACTION_LOG_DIR; import static org.apache.hadoop.ozone.OzoneConsts.DB_COMPACTION_SST_BACKUP_DIR; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.SNAPSHOT_INFO_TABLE; /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java index 9d28e7eb8ac..d95cf0f0696 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java @@ -35,7 +35,12 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; +/** + * Ozone Manager Snapshot Utilities. + */ public class OmSnapshotUtils { + + private OmSnapshotUtils(){} /** * Get the filename without the introductory metadata directory. * @@ -47,6 +52,12 @@ public static String truncateFileName(int truncateLength, Path file) { return file.toString().substring(truncateLength); } + /** + * Get the filename without the introductory metadata directory. + * + * @param file File whose INode is to be retrieved. + * @return INode for file. + */ @VisibleForTesting public static Object getINode(Path file) throws IOException { return Files.readAttributes(file, BasicFileAttributes.class).fileKey(); From 992f4d4d8e8bb85550b925a5452c55645b3a974e Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 27 Mar 2023 21:39:43 -0700 Subject: [PATCH 115/120] checkstyle --- .../org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java | 4 ++-- .../org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java index 3e6b7635671..24033680a73 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java @@ -32,14 +32,14 @@ /** * RocksDB Checkpoint Utilities. */ -public class RDBCheckpointUtils { +public final class RDBCheckpointUtils { static final Logger LOG = LoggerFactory.getLogger(RDBCheckpointUtils.class); private static final Duration POLL_DELAY_DURATION = Duration.ZERO; private static final Duration POLL_INTERVAL_DURATION = Duration.ofMillis(100); private static final Duration POLL_MAX_DURATION = Duration.ofSeconds(5); - private RDBCheckpointUtils(){} + private RDBCheckpointUtils() { } /** * Wait for checkpoint directory to be created for 5 secs with 100 millis diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java index d95cf0f0696..b6715436fcb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java @@ -38,9 +38,9 @@ /** * Ozone Manager Snapshot Utilities. */ -public class OmSnapshotUtils { +public final class OmSnapshotUtils { - private OmSnapshotUtils(){} + private OmSnapshotUtils() { } /** * Get the filename without the introductory metadata directory. * From 79b8936531924c5def9cc40b5b79f743d0481174 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 27 Mar 2023 22:18:03 -0700 Subject: [PATCH 116/120] fixed comment --- .../org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java index b6715436fcb..eb16f6e372f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java @@ -41,6 +41,7 @@ public final class OmSnapshotUtils { private OmSnapshotUtils() { } + /** * Get the filename without the introductory metadata directory. * @@ -53,7 +54,7 @@ public static String truncateFileName(int truncateLength, Path file) { } /** - * Get the filename without the introductory metadata directory. + * Get the INode for file * * @param file File whose INode is to be retrieved. * @return INode for file. From 27c5a9a83853d8f57798637e43523ecf9716937e Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 27 Mar 2023 23:39:55 -0700 Subject: [PATCH 117/120] trigger new CI check From d94e40640cec7e40da7eb9db7a304bd34f748d42 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 29 Mar 2023 07:39:38 -0700 Subject: [PATCH 118/120] checkstyle --- .../org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java index eb16f6e372f..9aef593af85 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java @@ -54,7 +54,7 @@ public static String truncateFileName(int truncateLength, Path file) { } /** - * Get the INode for file + * Get the INode for file. * * @param file File whose INode is to be retrieved. * @return INode for file. From 59b5e52c9f4282bdb8e2e59e60193e1fcc4a96f4 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 29 Mar 2023 14:08:40 -0700 Subject: [PATCH 119/120] fix checkstyle and TestOMSnapshotDAG --- .../hadoop/ozone/freon/TestOMSnapshotDAG.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java index 1cb16a481ca..e2ddfcdd7f9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java @@ -29,8 +29,8 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -62,9 +62,10 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; +import static org.apache.hadoop.ozone.OzoneConsts.DB_COMPACTION_LOG_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.DB_COMPACTION_SST_BACKUP_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DIR; /** * Tests Freon, with MiniOzoneCluster. @@ -77,9 +78,6 @@ public class TestOMSnapshotDAG { private static OzoneConfiguration conf; private static ObjectStore store; private static OzoneClient client; - private final File metaDir = OMStorage.getOmDbDir(conf); - private final String compactionLogDirName = "compaction-log"; - private final String sstBackUpDirName = "compaction-sst-backup"; /** * Create a MiniDFSCluster for testing. @@ -127,9 +125,7 @@ public static void shutdown() { } private String getDBCheckpointAbsolutePath(SnapshotInfo snapshotInfo) { - return metaDir + OM_KEY_PREFIX + - OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + - OM_DB_NAME + snapshotInfo.getCheckpointDirName(); + return OmSnapshotManager.getSnapshotPath(conf, snapshotInfo); } private static String getSnapshotDBKey(String volumeName, String bucketName, @@ -276,7 +272,7 @@ public void testDAGReconstruction() } @Test - public void testSkipTrackingWithZeroSnapshot() throws IOException { + public void testSkipTrackingWithZeroSnapshot() { // Verify that the listener correctly skips compaction tracking // when there is no snapshot in SnapshotInfoTable. @@ -302,7 +298,8 @@ public void testSkipTrackingWithZeroSnapshot() throws IOException { String omMetadataDir = cluster.getOzoneManager().getConfiguration().get(OZONE_METADATA_DIRS); // Verify that no compaction log entry has been written - Path logPath = Paths.get(omMetadataDir, compactionLogDirName); + Path logPath = Paths.get(omMetadataDir, OM_SNAPSHOT_DIFF_DIR, + DB_COMPACTION_LOG_DIR); File[] fileList = logPath.toFile().listFiles(); Assertions.assertNotNull(fileList); for (File file : fileList) { @@ -311,7 +308,8 @@ public void testSkipTrackingWithZeroSnapshot() throws IOException { } } // Verify that no SST has been backed up - Path sstBackupPath = Paths.get(omMetadataDir, sstBackUpDirName); + Path sstBackupPath = Paths.get(omMetadataDir, OM_SNAPSHOT_DIFF_DIR, + DB_COMPACTION_SST_BACKUP_DIR); fileList = sstBackupPath.toFile().listFiles(); Assertions.assertNotNull(fileList); Assertions.assertEquals(0L, fileList.length); From 49dcc6edbbf53cded168e2a394c09d2f7a67f5b8 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Wed, 29 Mar 2023 19:11:45 -0700 Subject: [PATCH 120/120] trigger new CI check