From ddc19e9c1db02a04dcfd0a77d970f5afe9a0ea77 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Fri, 29 Dec 2023 19:25:23 +0100 Subject: [PATCH] HDDS-10007. Rename ManagedSstFileReader in rocksdb-checkpoint-differ (#5859) --- dev-support/ci/selective_ci_checks.bats | 25 +++++++++++-------- ...tFileReader.java => SstFileSetReader.java} | 8 +++--- ...eReader.java => TestSstFileSetReader.java} | 6 ++--- .../om/snapshot/SnapshotDiffManager.java | 6 ++--- .../om/snapshot/TestSnapshotDiffManager.java | 6 ++--- 5 files changed, 27 insertions(+), 24 deletions(-) rename hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/{ManagedSstFileReader.java => SstFileSetReader.java} (97%) rename hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/{TestManagedSstFileReader.java => TestSstFileSetReader.java} (98%) diff --git a/dev-support/ci/selective_ci_checks.bats b/dev-support/ci/selective_ci_checks.bats index cc14eeb0947..9fe1708c913 100644 --- a/dev-support/ci/selective_ci_checks.bats +++ b/dev-support/ci/selective_ci_checks.bats @@ -177,17 +177,20 @@ load bats-assert/load.bash assert_output -p needs-kubernetes-tests=false } -@test "native test in other module" { - run dev-support/ci/selective_ci_checks.sh 7d01cc14a6 - - assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","native","unit"]' - assert_output -p needs-build=true - assert_output -p needs-compile=true - assert_output -p needs-compose-tests=false - assert_output -p needs-dependency-check=false - assert_output -p needs-integration-tests=false - assert_output -p needs-kubernetes-tests=false -} +# disabled, because this test fails if +# hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestManagedSstFileReader.java +# is not present in the current tree (i.e. if file is renamed, moved or deleted) +#@test "native test in other module" { +# run dev-support/ci/selective_ci_checks.sh 7d01cc14a6 +# +# assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","native","unit"]' +# assert_output -p needs-build=true +# assert_output -p needs-compile=true +# assert_output -p needs-compose-tests=false +# assert_output -p needs-dependency-check=false +# assert_output -p needs-integration-tests=false +# assert_output -p needs-kubernetes-tests=false +#} @test "kubernetes only" { run dev-support/ci/selective_ci_checks.sh 5336bb9bd diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java similarity index 97% rename from hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java rename to hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java index 120b199c880..3b30ca5a146 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java @@ -46,16 +46,16 @@ import static java.nio.charset.StandardCharsets.UTF_8; /** - * ManagedSstFileReader provides an abstraction layer using which we can - * iterate over multiple underlying SST files transparently. + * Provides an abstraction layer using which we can iterate over multiple + * underlying SST files transparently. */ -public class ManagedSstFileReader { +public class SstFileSetReader { private final Collection sstFiles; private volatile long estimatedTotalKeys = -1; - public ManagedSstFileReader(final Collection sstFiles) { + public SstFileSetReader(final Collection sstFiles) { this.sstFiles = sstFiles; } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java similarity index 98% rename from hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestManagedSstFileReader.java rename to hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java index 588e54ad8b3..3b281ee09e7 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java @@ -58,7 +58,7 @@ /** * ManagedSstFileReader tests. */ -class TestManagedSstFileReader { +class TestSstFileSetReader { @TempDir private File tempDir; @@ -146,7 +146,7 @@ public void testGetKeyStream(int numberOfFiles) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); try (Stream keyStream = - new ManagedSstFileReader(files).getKeyStream( + new SstFileSetReader(files).getKeyStream( lowerBound.orElse(null), upperBound.orElse(null))) { keyStream.forEach(key -> { Assertions.assertEquals(1, keysInBoundary.get(key)); @@ -194,7 +194,7 @@ public void testGetKeyStreamWithTombstone(int numberOfFiles) .orElse(true)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - try (Stream keyStream = new ManagedSstFileReader(files) + try (Stream keyStream = new SstFileSetReader(files) .getKeyStreamWithTombstone(sstDumpTool, lowerBound.orElse(null), upperBound.orElse(null))) { keyStream.forEach( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 2c0a564306b..99134799de9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -56,7 +56,7 @@ import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus; import org.apache.hadoop.util.ClosableIterator; import org.apache.logging.log4j.util.Strings; -import org.apache.ozone.rocksdb.util.ManagedSstFileReader; +import org.apache.ozone.rocksdb.util.SstFileSetReader; import org.apache.ozone.rocksdb.util.RdbUtil; import org.apache.ozone.rocksdiff.DifferSnapshotInfo; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; @@ -1083,7 +1083,7 @@ void addToObjectIdMap(Table fsTable, String tablePrefix = getTablePrefix(tablePrefixes, fsTable.getName()); boolean isDirectoryTable = fsTable.getName().equals(DIRECTORY_TABLE); - ManagedSstFileReader sstFileReader = new ManagedSstFileReader(deltaFiles); + SstFileSetReader sstFileReader = new SstFileSetReader(deltaFiles); validateEstimatedKeyChangesAreInLimits(sstFileReader); String sstFileReaderLowerBound = tablePrefix; String sstFileReaderUpperBound = null; @@ -1204,7 +1204,7 @@ Set getDeltaFiles(OmSnapshot fromSnapshot, } private void validateEstimatedKeyChangesAreInLimits( - ManagedSstFileReader sstFileReader + SstFileSetReader sstFileReader ) throws RocksDBException, IOException { if (sstFileReader.getEstimatedTotalKeys() > maxAllowedKeyChangesForASnapDiff) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 28af68e2539..e453ebf7b55 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -63,7 +63,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ClosableIterator; import org.apache.hadoop.util.ExitUtil; -import org.apache.ozone.rocksdb.util.ManagedSstFileReader; +import org.apache.ozone.rocksdb.util.SstFileSetReader; import org.apache.ozone.rocksdb.util.RdbUtil; import org.apache.ozone.rocksdiff.DifferSnapshotInfo; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; @@ -651,8 +651,8 @@ public void testObjectIdMapWithTombstoneEntries(boolean nativeLibraryLoaded, .map(i -> (i + 100) + "/key" + i).collect(Collectors.toSet()); // Mocking SSTFileReader functions to return the above keys list. - try (MockedConstruction mockedSSTFileReader = - Mockito.mockConstruction(ManagedSstFileReader.class, + try (MockedConstruction mockedSSTFileReader = + Mockito.mockConstruction(SstFileSetReader.class, (mock, context) -> { when(mock.getKeyStreamWithTombstone(any(), any(), any())) .thenReturn(keysIncludingTombstones.stream());