diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java index f3e54f375cb..c25c0a40c53 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java @@ -40,4 +40,47 @@ public interface SpaceUsageSource { long getCapacity(); long getAvailable(); + + default SpaceUsageSource snapshot() { + return new Fixed(getCapacity(), getAvailable(), getUsedSpace()); + } + + SpaceUsageSource UNKNOWN = new Fixed(0, 0, 0); + + /** + * A static source of space usage. Can be a point in time snapshot of a + * real volume usage, or can be used for testing. + */ + final class Fixed implements SpaceUsageSource { + + private final long capacity; + private final long available; + private final long used; + + Fixed(long capacity, long available, long used) { + this.capacity = capacity; + this.available = available; + this.used = used; + } + + @Override + public long getCapacity() { + return capacity; + } + + @Override + public long getAvailable() { + return available; + } + + @Override + public long getUsedSpace() { + return used; + } + + @Override + public SpaceUsageSource snapshot() { + return this; // immutable + } + } } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageSource.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageSource.java index 4055f080b99..26c861dc68b 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageSource.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageSource.java @@ -36,35 +36,7 @@ public static SpaceUsageSource fixed(long capacity, long available) { public static SpaceUsageSource fixed(long capacity, long available, long used) { - return new Fixed(capacity, available, used); - } - - private static final class Fixed implements SpaceUsageSource { - - private final long capacity; - private final long available; - private final long used; - - Fixed(long capacity, long available, long used) { - this.capacity = capacity; - this.available = available; - this.used = used; - } - - @Override - public long getCapacity() { - return capacity; - } - - @Override - public long getAvailable() { - return available; - } - - @Override - public long getUsedSpace() { - return used; - } + return new SpaceUsageSource.Fixed(capacity, available, used); } private MockSpaceUsageSource() { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index 1b0ef29c770..eeb3548d5f8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.fs.SpaceUsageSource; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto; @@ -78,7 +79,6 @@ import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.malformedRequest; import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest; import static org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; -import static org.apache.hadoop.ozone.container.common.volume.VolumeUsage.PrecomputedVolumeSpace; /** * Ozone Container dispatcher takes a call from the netty server and routes it @@ -585,8 +585,8 @@ private boolean isVolumeFull(Container container) { .orElse(Boolean.FALSE); if (isOpen) { HddsVolume volume = container.getContainerData().getVolume(); - PrecomputedVolumeSpace precomputedVolumeSpace = - volume.getPrecomputedVolumeSpace(); + SpaceUsageSource precomputedVolumeSpace = + volume.getCurrentUsage(); long volumeCapacity = precomputedVolumeSpace.getCapacity(); long volumeFreeSpaceToSpare = VolumeUsage.getMinVolumeFreeSpace(conf, volumeCapacity); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java index fc329ebeb0e..44ae1c0e795 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory; +import org.apache.hadoop.hdds.fs.SpaceUsageSource; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.checker.Checkable; import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult; @@ -50,7 +51,6 @@ import java.util.stream.Stream; import static org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion.getLatestVersion; -import static org.apache.hadoop.ozone.container.common.volume.VolumeUsage.PrecomputedVolumeSpace; /** @@ -456,14 +456,14 @@ public long getAvailable() { } - public long getAvailable(PrecomputedVolumeSpace precomputedVolumeSpace) { + public long getAvailable(SpaceUsageSource precomputedVolumeSpace) { return volumeInfo.map(info -> info.getAvailable(precomputedVolumeSpace)) .orElse(0L); } - public PrecomputedVolumeSpace getPrecomputedVolumeSpace() { - return volumeInfo.map(VolumeInfo::getPrecomputedVolumeSpace) - .orElse(new PrecomputedVolumeSpace(0L, 0L)); + public SpaceUsageSource getCurrentUsage() { + return volumeInfo.map(VolumeInfo::getCurrentUsage) + .orElse(SpaceUsageSource.UNKNOWN); } public long getUsedSpace() { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java index 61c0b422c7f..6ee35ba6b09 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java @@ -29,10 +29,10 @@ import org.apache.hadoop.hdds.fs.SpaceUsageCheckParams; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.fs.SpaceUsageSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.hadoop.ozone.container.common.volume.VolumeUsage.PrecomputedVolumeSpace; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT_DEFAULT; @@ -240,13 +240,13 @@ public long getAvailable() { return Math.max(Math.min(avail, usage.getAvailable()), 0); } - public long getAvailable(PrecomputedVolumeSpace precomputedValues) { + public long getAvailable(SpaceUsageSource precomputedValues) { long avail = precomputedValues.getCapacity() - usage.getUsedSpace(); return Math.max(Math.min(avail, usage.getAvailable(precomputedValues)), 0); } - public PrecomputedVolumeSpace getPrecomputedVolumeSpace() { - return usage.getPrecomputedVolumeSpace(); + public SpaceUsageSource getCurrentUsage() { + return usage.snapshot(); } public void incrementUsedSpace(long usedSpace) { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java index 57cf0a8b9dd..b2a66ba16b4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java @@ -65,7 +65,7 @@ public long getAvailable() { return source.getAvailable() - getRemainingReserved(); } - public long getAvailable(PrecomputedVolumeSpace precomputedVolumeSpace) { + public long getAvailable(SpaceUsageSource precomputedVolumeSpace) { long available = precomputedVolumeSpace.getAvailable(); return available - getRemainingReserved(precomputedVolumeSpace); } @@ -75,6 +75,11 @@ public long getUsedSpace() { return source.getUsedSpace(); } + @Override + public SpaceUsageSource snapshot() { + return source.snapshot(); + } + public void incrementUsedSpace(long usedSpace) { source.incrementUsedSpace(usedSpace); } @@ -94,7 +99,7 @@ private long getOtherUsed() { return Math.max(totalUsed - source.getUsedSpace(), 0L); } - private long getOtherUsed(PrecomputedVolumeSpace precomputedVolumeSpace) { + private long getOtherUsed(SpaceUsageSource precomputedVolumeSpace) { long totalUsed = precomputedVolumeSpace.getCapacity() - precomputedVolumeSpace.getAvailable(); return Math.max(totalUsed - source.getUsedSpace(), 0L); @@ -105,7 +110,7 @@ private long getRemainingReserved() { } private long getRemainingReserved( - PrecomputedVolumeSpace precomputedVolumeSpace) { + SpaceUsageSource precomputedVolumeSpace) { return Math.max(reservedInBytes - getOtherUsed(precomputedVolumeSpace), 0L); } @@ -169,33 +174,4 @@ public static boolean hasVolumeEnoughSpace(long volumeAvailableSpace, return (volumeAvailableSpace - volumeCommittedBytesCount) > Math.max(requiredSpace, volumeFreeSpaceToSpare); } - - /** - * Class representing precomputed space values of a volume. - * This class is intended to store precomputed values, such as capacity - * and available space of a volume, to avoid recalculating these - * values multiple times and to make method signatures simpler. - */ - public static class PrecomputedVolumeSpace { - private final long capacity; - private final long available; - - public PrecomputedVolumeSpace(long capacity, long available) { - this.capacity = capacity; - this.available = available; - } - - public long getCapacity() { - return capacity; - } - - public long getAvailable() { - return available; - } - } - - public PrecomputedVolumeSpace getPrecomputedVolumeSpace() { - return new PrecomputedVolumeSpace(source.getCapacity(), - source.getAvailable()); - } }