From 83113f7befc46df676d9798cad23a06e952b464a Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Tue, 7 May 2024 22:42:13 -0700 Subject: [PATCH 1/2] HDDS-10696. Fixed ozone integration test fails because of empty snapshot installation --- .../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 2 -- .../apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 10 ++++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestAddRemoveOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestAddRemoveOzoneManager.java index da0f82d4707..463f44bf058 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestAddRemoveOzoneManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestAddRemoveOzoneManager.java @@ -47,7 +47,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.tag.Flaky; import org.apache.ratis.grpc.server.GrpcLogAppender; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; @@ -184,7 +183,6 @@ private List testBootstrapOMs(int numNewOMs) throws Exception { * OM. */ @Test - @Flaky("HDDS-7880") public void testBootstrap() throws Exception { setupCluster(1); OzoneManager oldOM = cluster.getOzoneManager(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index cc8acc48340..265625ea993 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.utils.DBCheckpointServlet; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.RDBCheckpointUtils; +import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneConsts; @@ -317,8 +318,7 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint, // Get the snapshot files. Set snapshotPaths = waitForSnapshotDirs(checkpoint); - Path snapshotDir = Paths.get(OMStorage.getOmDbDir(getConf()).toString(), - OM_SNAPSHOT_DIR); + Path snapshotDir = Paths.get(getSnapshotsParentDir()).getParent(); if (!processDir(snapshotDir, copyFiles, hardLinkFiles, sstFilesToExclude, snapshotPaths, excluded, copySize, null)) { return false; @@ -635,6 +635,12 @@ private OzoneConfiguration getConf() { .getConfiguration(); } + private String getSnapshotsParentDir() { + OzoneManager om = (OzoneManager) getServletContext().getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE); + RDBStore store = (RDBStore) om.getMetadataManager().getStore(); + return store.getSnapshotsParentDir(); + } + @Override public BootstrapStateHandler.Lock getBootstrapStateLock() { return lock; From de0ef068e9742d1ec0c1345bc88b68583a673258 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Wed, 8 May 2024 16:19:24 -0700 Subject: [PATCH 2/2] Added a comment and renamed the function --- .../apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 265625ea993..6d2832c9138 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -318,7 +318,7 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint, // Get the snapshot files. Set snapshotPaths = waitForSnapshotDirs(checkpoint); - Path snapshotDir = Paths.get(getSnapshotsParentDir()).getParent(); + Path snapshotDir = getSnapshotDir(); if (!processDir(snapshotDir, copyFiles, hardLinkFiles, sstFilesToExclude, snapshotPaths, excluded, copySize, null)) { return false; @@ -635,10 +635,13 @@ private OzoneConfiguration getConf() { .getConfiguration(); } - private String getSnapshotsParentDir() { + private Path getSnapshotDir() { OzoneManager om = (OzoneManager) getServletContext().getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE); RDBStore store = (RDBStore) om.getMetadataManager().getStore(); - return store.getSnapshotsParentDir(); + // store.getSnapshotsParentDir() returns path to checkpointState (e.g. /db.snapshots/checkpointState) + // But we need to return path till db.snapshots which contains checkpointState and diffState. + // So that whole snapshots and compaction information can be transferred to follower. + return Paths.get(store.getSnapshotsParentDir()).getParent(); } @Override