From d9bff79734e430e1ebcc47c36c6a6117a6c2789b Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran <47532440+swamirishi@users.noreply.github.com> Date: Sun, 17 Nov 2024 20:44:55 -0800 Subject: [PATCH] HDDS-11650. ContainerId list to track all containers created in a datanode (#4) * HDDS-11650. ContainerId list to track all containers created in a datanode Change-Id: I94fd413a2d778ac5d86a7da5126cf3d1cac8113a * HDDS-11650. Fix test cases Change-Id: I22654091edbd3a11c585aa95ca2b554eba0f9d95 * HDDS-11650. Add comments Change-Id: Icaa5ae0b29ec0ffccf5914bec0fd6ed6ae117219 * HDDS-11650. Fix tests Change-Id: I995fc25b93f16aa859eeb8f0418aa774e3719330 * HDDS-11650. Remove from rocskdb Change-Id: Ibeadc9330185f699e4cf1d9c1c8631d1af52683e * HDDS-11650. Fix checkstyle Change-Id: I5ac6a685a49e79be5ea43717294dd649383433f2 * HDDS-11650. Fix Issues Change-Id: I18f48f9d97b0cc16a3c97a3137ee01ebda4fcbec * HDDS-11650. Fix checkstyle & rat Change-Id: I16cec52ea1c2853c80ee9a6e3279a23408d05651 * HDDS-11650. Fix checkstyle & rat Change-Id: Icf779e2bff8ace1721b529e3c89edbe9effa9989 * HDDS-11650. Fix tests failures Change-Id: I485646e86105a8a1bab6b638262669fc5f92d94d * HDDS-11650. Fix tests failures Change-Id: I03ab7dd188ae39248ca889f40b9490eb2870579f * HDDS-11650. Fix MasterVolumeMetaStore cache Change-Id: I82647ef09edc6fd9432652d911bf2ff4bccf25a5 * HDDS-11650. Fix MasterVolumeMetaStore cache Change-Id: I73091fc280dea5ad447b9df8bb0a1877d8f1ff35 * HDDS-11650. Fix MasterVolumeMetaStore cache Change-Id: Ife63a4ab2a69869cce9d1c407bfdeba2540d2482 * HDDS-11650. Fix acceptance tests Change-Id: Ic9fe75b9efe885080e3ad440f132eb0100c41a17 * HDDS-11650. Fix acceptance tests Change-Id: I5a8e092d8fb751a2ca69256740df59edd59b9b95 * HDDS-11650. Add an integration test to test dn restarts with missing containers Change-Id: Ic67537ed852920d8945430665e22eeddc7350d6e * HDDS-11650. Address review comments Change-Id: Icf8b45e0c2de6d353f3f880c441de7d7a6138009 * HDDS-11650. Address review comments Change-Id: I7ead428f7ff82968a0f1e5058bbf65f3b807bdb9 * HDDS-11650. Reduce number of files changed Change-Id: I4476c42d2c8c11af64fc5808eafcecc9046bd2e8 * HDDS-11650. Fix checkstyle Change-Id: I33f2a74daa2812d227e985b1a0738bc0be99e6e7 --- .../hadoop/hdds/utils/db/Proto2EnumCodec.java | 98 +++++++++ .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../container/common/impl/ContainerSet.java | 83 ++++++- .../container/common/impl/HddsDispatcher.java | 6 +- .../common/interfaces/BaseDBHandle.java | 56 +++++ .../container/common/interfaces/DBHandle.java | 25 +-- .../common/utils/BaseReferenceCountedDB.java | 92 ++++++++ .../common/utils/ReferenceCountedHandle.java | 56 +++++ .../common/volume/MutableVolumeSet.java | 7 +- .../container/keyvalue/KeyValueHandler.java | 17 +- .../metadata/AbstractDatanodeStore.java | 202 +++++------------- .../container/metadata/AbstractRDBStore.java | 135 ++++++++++++ .../container/metadata/AbstractStore.java | 73 +++++++ .../container/metadata/DatanodeStore.java | 49 +---- .../metadata/MasterVolumeDBDefinition.java | 72 +++++++ .../metadata/MasterVolumeMetadataStore.java | 88 ++++++++ .../container/metadata/MetadataStore.java | 35 +++ .../container/ozoneimpl/ContainerReader.java | 2 +- .../OnDemandContainerDataScanner.java | 3 + .../container/ozoneimpl/OzoneContainer.java | 30 ++- .../replication/ContainerImporter.java | 2 +- .../volume/TestVolumeSetDiskChecks.java | 1 + .../hadoop/hdds/utils/VoidCallable.java | 26 +++ .../hadoop/hdds/utils/db/DBStoreBuilder.java | 9 +- .../hadoop/hdds/utils/db/DBTestUtils.java | 142 ++++++++++++ .../main/compose/compatibility/docker-config | 2 +- .../main/compose/ozone-balancer/docker-config | 2 +- .../src/main/compose/ozone-csi/docker-config | 2 +- .../src/main/compose/ozone-ha/docker-config | 2 +- .../main/compose/ozone-om-ha/docker-config | 2 +- .../compose/ozone-om-prepare/docker-config | 2 +- .../main/compose/ozone-topology/docker-config | 2 +- .../dist/src/main/compose/ozone/docker-config | 2 +- .../main/compose/ozoneblockade/docker-config | 2 +- .../main/compose/ozonescripts/docker-config | 2 +- .../main/compose/ozonesecure-ha/docker-config | 2 +- .../main/compose/ozonesecure-mr/docker-config | 2 +- .../main/compose/ozonesecure/docker-config | 2 +- .../src/main/compose/restart/docker-config | 2 +- .../compose/upgrade/compose/ha/docker-config | 2 +- .../upgrade/compose/non-ha/docker-config | 2 +- .../upgrade/compose/om-ha/docker-config | 2 +- .../src/main/compose/xcompat/docker-config | 2 +- .../ozoneimpl/TestOzoneContainer.java | 138 +++++++++++- .../ozone/debug/DBDefinitionFactory.java | 4 +- .../debug/container/ContainerCommands.java | 2 +- .../freon/ClosedContainerReplicator.java | 87 ++++---- 47 files changed, 1270 insertions(+), 307 deletions(-) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto2EnumCodec.java create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/BaseDBHandle.java create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/BaseReferenceCountedDB.java create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ReferenceCountedHandle.java create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractRDBStore.java create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractStore.java create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MasterVolumeDBDefinition.java create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MasterVolumeMetadataStore.java create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MetadataStore.java create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/VoidCallable.java create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBTestUtils.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto2EnumCodec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto2EnumCodec.java new file mode 100644 index 00000000000..d206b17a9b1 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto2EnumCodec.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.utils.db; + +import org.apache.ratis.thirdparty.com.google.protobuf.ProtocolMessageEnum; +import jakarta.annotation.Nonnull; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Codecs to serialize/deserialize Protobuf v2 enums. + */ +public final class Proto2EnumCodec + implements Codec { + private static final ConcurrentMap, + Codec> CODECS + = new ConcurrentHashMap<>(); + private static final IntegerCodec INTEGER_CODEC = IntegerCodec.get(); + + /** + * @return the {@link Codec} for the given class. + */ + public static Codec get(T t) { + final Codec codec = CODECS.computeIfAbsent(t.getClass(), + key -> new Proto2EnumCodec<>(t)); + return (Codec) codec; + } + + private final Class clazz; + + private Proto2EnumCodec(M m) { + this.clazz = (Class) m.getClass(); + } + + @Override + public Class getTypeClass() { + return clazz; + } + + @Override + public boolean supportCodecBuffer() { + return INTEGER_CODEC.supportCodecBuffer(); + } + + @Override + public CodecBuffer toCodecBuffer(@Nonnull M value, + CodecBuffer.Allocator allocator) throws IOException { + return INTEGER_CODEC.toCodecBuffer(value.getNumber(), allocator); + } + + private M parseFrom(Integer value) throws IOException { + try { + return (M) this.clazz.getDeclaredMethod("forNumber", int.class).invoke(null, value); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new IOException(e); + } + } + + @Override + public M fromCodecBuffer(@Nonnull CodecBuffer buffer) + throws IOException { + return parseFrom(INTEGER_CODEC.fromCodecBuffer(buffer)); + } + + @Override + public byte[] toPersistedFormat(M value) { + return INTEGER_CODEC.toPersistedFormat(value.getNumber()); + } + + @Override + public M fromPersistedFormat(byte[] bytes) throws IOException { + return parseFrom(INTEGER_CODEC.fromPersistedFormat(bytes)); + } + + @Override + public M copyObject(M message) { + // proto messages are immutable + return message; + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index e483feba98d..0eeedce1c64 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -122,6 +122,7 @@ public final class OzoneConsts { public static final String OM_DB_BACKUP_PREFIX = "om.db.backup."; public static final String SCM_DB_BACKUP_PREFIX = "scm.db.backup."; public static final String CONTAINER_DB_NAME = "container.db"; + public static final String CONTAINER_META_DB_NAME = "container_meta.db"; public static final String STORAGE_DIR_CHUNKS = "chunks"; public static final String OZONE_DB_CHECKPOINT_REQUEST_FLUSH = diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java index 5335021da9e..a49f429e056 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java @@ -23,8 +23,12 @@ import com.google.common.collect.ImmutableMap; import com.google.protobuf.Message; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State; + import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; +import org.apache.hadoop.hdds.utils.db.DBTestUtils; +import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.utils.ContainerLogger; @@ -65,10 +69,24 @@ public class ContainerSet implements Iterable> { new ConcurrentSkipListMap<>(); private Clock clock; private long recoveringTimeout; + private final Table containerIdsTable; + @VisibleForTesting public ContainerSet(long recoveringTimeout) { + this(DBTestUtils.getInMemoryTableForTest(), recoveringTimeout); + } + + public ContainerSet(Table continerIdsTable, long recoveringTimeout) { + this(continerIdsTable, recoveringTimeout, false); + } + + public ContainerSet(Table continerIdsTable, long recoveringTimeout, boolean readOnly) { this.clock = Clock.system(ZoneOffset.UTC); + this.containerIdsTable = continerIdsTable; this.recoveringTimeout = recoveringTimeout; + if (!readOnly && containerIdsTable == null) { + throw new IllegalArgumentException("Container table cannot be null when container set is not read only"); + } } public long getCurrentTime() { @@ -85,22 +103,46 @@ public void setRecoveringTimeout(long recoveringTimeout) { this.recoveringTimeout = recoveringTimeout; } + public boolean addContainer(Container container) throws StorageContainerException { + return addContainer(container, false); + } + + public void validateContainerIsMissing(long containerId, State state) throws StorageContainerException { + if (missingContainerSet.contains(containerId)) { + throw new StorageContainerException(String.format("Container with container Id %d with state : %s is missing in" + + " the DN.", containerId, state), + ContainerProtos.Result.CONTAINER_MISSING); + } + } + /** * Add Container to container map. * @param container container to be added * @return If container is added to containerMap returns true, otherwise * false */ - public boolean addContainer(Container container) throws + public boolean addContainer(Container container, boolean overwriteMissingContainers) throws StorageContainerException { Preconditions.checkNotNull(container, "container cannot be null"); long containerId = container.getContainerData().getContainerID(); + State containerState = container.getContainerData().getState(); + if (!overwriteMissingContainers) { + validateContainerIsMissing(containerId, containerState); + } if (containerMap.putIfAbsent(containerId, container) == null) { if (LOG.isDebugEnabled()) { LOG.debug("Container with container Id {} is added to containerMap", containerId); } + try { + if (containerIdsTable != null) { + containerIdsTable.put(containerId, containerState); + } + } catch (IOException e) { + throw new StorageContainerException(e, ContainerProtos.Result.IO_EXCEPTION); + } + missingContainerSet.remove(containerId); // wish we could have done this from ContainerData.setState container.getContainerData().commitSpace(); if (container.getContainerData().getState() == RECOVERING) { @@ -122,21 +164,40 @@ public boolean addContainer(Container container) throws * @return Container */ public Container getContainer(long containerId) { - Preconditions.checkState(containerId >= 0, - "Container Id cannot be negative."); + Preconditions.checkState(containerId >= 0, "Container Id cannot be negative."); return containerMap.get(containerId); } + public boolean removeContainer(long containerId) throws StorageContainerException { + return removeContainer(containerId, false, true); + } + /** * Removes the Container matching with specified containerId. * @param containerId ID of the container to remove * @return If container is removed from containerMap returns true, otherwise * false */ - public boolean removeContainer(long containerId) { + public boolean removeContainer(long containerId, boolean markMissing, boolean removeFromDB) + throws StorageContainerException { Preconditions.checkState(containerId >= 0, "Container Id cannot be negative."); + //We need to add to missing container set before removing containerMap since there could be write chunk operation + // that could recreate the container in another volume if we remove it from the map before adding to missing + // container. + if (markMissing) { + missingContainerSet.add(containerId); + } Container removed = containerMap.remove(containerId); + if (removeFromDB) { + try { + if (containerIdsTable != null) { + containerIdsTable.delete(containerId); + } + } catch (IOException e) { + throw new StorageContainerException(e, ContainerProtos.Result.IO_EXCEPTION); + } + } if (removed == null) { LOG.debug("Container with containerId {} is not present in " + "containerMap", containerId); @@ -190,20 +251,20 @@ public int containerCount() { * * @param context StateContext */ - public void handleVolumeFailures(StateContext context) { + public void handleVolumeFailures(StateContext context) throws StorageContainerException { AtomicBoolean failedVolume = new AtomicBoolean(false); AtomicInteger containerCount = new AtomicInteger(0); - containerMap.values().forEach(c -> { + for (Container c : containerMap.values()) { ContainerData data = c.getContainerData(); if (data.getVolume().isFailed()) { - removeContainer(data.getContainerID()); + removeContainer(data.getContainerID(), true, false); LOG.debug("Removing Container {} as the Volume {} " + - "has failed", data.getContainerID(), data.getVolume()); + "has failed", data.getContainerID(), data.getVolume()); failedVolume.set(true); containerCount.incrementAndGet(); ContainerLogger.logLost(data, "Volume failure"); } - }); + } if (failedVolume.get()) { try { @@ -362,6 +423,10 @@ public Set getMissingContainerSet() { return missingContainerSet; } + public Table getContainerIdsTable() { + return containerIdsTable; + } + /** * Builds the missing container set by taking a diff between total no * containers actually found and number of containers which actually diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index 5fc97184155..d86cf4db163 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -175,7 +175,8 @@ private boolean canIgnoreException(Result result) { case CONTAINER_UNHEALTHY: case CLOSED_CONTAINER_IO: case DELETE_ON_OPEN_CONTAINER: - case UNSUPPORTED_REQUEST: // Blame client for sending unsupported request. + case UNSUPPORTED_REQUEST: + case CONTAINER_MISSING:// Blame client for sending unsupported request. return true; default: return false; @@ -276,7 +277,8 @@ private ContainerCommandResponseProto dispatchRequest( getMissingContainerSet().remove(containerID); } } - if (getMissingContainerSet().contains(containerID)) { + if (cmdType != Type.CreateContainer && !HddsUtils.isReadOnly(msg) + && getMissingContainerSet().contains(containerID)) { StorageContainerException sce = new StorageContainerException( "ContainerID " + containerID + " has been lost and cannot be recreated on this DataNode", diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/BaseDBHandle.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/BaseDBHandle.java new file mode 100644 index 00000000000..aaaa0128109 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/BaseDBHandle.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.common.interfaces; + +import org.apache.hadoop.ozone.container.metadata.AbstractStore; + +import java.io.Closeable; + +/** + * DB handle abstract class. + */ +public abstract class BaseDBHandle implements Closeable { + + private final STORE store; + private final String containerDBPath; + + public BaseDBHandle(STORE store, String containerDBPath) { + this.store = store; + this.containerDBPath = containerDBPath; + } + + public STORE getStore() { + return this.store; + } + + public String getContainerDBPath() { + return this.containerDBPath; + } + + public boolean cleanup() { + return true; + } + + @Override + public String toString() { + return "DBHandle{" + + "containerDBPath='" + containerDBPath + '\'' + + ", store=" + store + + '}'; + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/DBHandle.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/DBHandle.java index 839a112ed9b..aea67917b2f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/DBHandle.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/DBHandle.java @@ -19,30 +19,11 @@ import org.apache.hadoop.ozone.container.metadata.DatanodeStore; -import java.io.Closeable; - /** - * DB handle abstract class. + * DB handle abstract class for datanode store. */ -public abstract class DBHandle implements Closeable { - - private final DatanodeStore store; - private final String containerDBPath; - +public abstract class DBHandle extends BaseDBHandle { public DBHandle(DatanodeStore store, String containerDBPath) { - this.store = store; - this.containerDBPath = containerDBPath; - } - - public DatanodeStore getStore() { - return this.store; - } - - public String getContainerDBPath() { - return this.containerDBPath; - } - - public boolean cleanup() { - return true; + super(store, containerDBPath); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/BaseReferenceCountedDB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/BaseReferenceCountedDB.java new file mode 100644 index 00000000000..b2d9fbda4ba --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/BaseReferenceCountedDB.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.common.utils; + +import com.google.common.base.Preconditions; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.hadoop.ozone.container.common.interfaces.BaseDBHandle; +import org.apache.hadoop.ozone.container.metadata.AbstractStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Class to implement reference counting over instances of a db handle. + */ +public class BaseReferenceCountedDB extends BaseDBHandle { + private static final Logger LOG = + LoggerFactory.getLogger(BaseReferenceCountedDB.class); + private final AtomicInteger referenceCount; + + public BaseReferenceCountedDB(STORE store, String containerDBPath) { + super(store, containerDBPath); + this.referenceCount = new AtomicInteger(0); + } + + public void incrementReference() { + this.referenceCount.incrementAndGet(); + if (LOG.isTraceEnabled()) { + LOG.trace("IncRef {} to refCnt {}, stackTrace: {}", getContainerDBPath(), + referenceCount.get(), ExceptionUtils.getStackTrace(new Throwable())); + } + } + + public void decrementReference() { + int refCount = this.referenceCount.decrementAndGet(); + Preconditions.checkArgument(refCount >= 0, "refCount:", refCount); + if (LOG.isTraceEnabled()) { + LOG.trace("DecRef {} to refCnt {}, stackTrace: {}", getContainerDBPath(), + referenceCount.get(), ExceptionUtils.getStackTrace(new Throwable())); + } + } + + public boolean cleanup() { + if (getStore() != null && getStore().isClosed() + || referenceCount.get() == 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("Close {} refCnt {}", getContainerDBPath(), + referenceCount.get()); + } + try { + getStore().stop(); + return true; + } catch (Exception e) { + LOG.error("Error closing DB. Container: " + getContainerDBPath(), e); + return false; + } + } else { + return false; + } + } + + @Override + public void close() throws IOException { + decrementReference(); + } + + /** + * Returns if the underlying DB is closed. This call is threadsafe. + * @return true if the DB is closed. + */ + public boolean isClosed() { + return getStore().isClosed(); + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ReferenceCountedHandle.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ReferenceCountedHandle.java new file mode 100644 index 00000000000..ca24c99ab08 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ReferenceCountedHandle.java @@ -0,0 +1,56 @@ +package org.apache.hadoop.ozone.container.common.utils; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +import org.apache.hadoop.ozone.container.metadata.AbstractStore; + +import java.io.Closeable; + +/** + * Class enclosing a reference counted handle to DBStore. + */ +public class ReferenceCountedHandle implements Closeable { + private final BaseReferenceCountedDB dbHandle; + private volatile boolean isClosed; + + //Provide a handle with an already incremented reference. + public ReferenceCountedHandle(BaseReferenceCountedDB dbHandle) { + this.dbHandle = dbHandle; + this.isClosed = false; + } + + public STORE getStore() { + return dbHandle.getStore(); + } + + @Override + public void close() { + if (!isClosed) { + synchronized (this) { + if (!isClosed) { + if (!dbHandle.isClosed()) { + dbHandle.decrementReference(); + dbHandle.cleanup(); + } + this.isClosed = true; + } + } + } + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java index e195b127d49..426012f3765 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory; import org.apache.hadoop.hdds.utils.HddsServerUtil; +import org.apache.hadoop.hdds.utils.VoidCallable; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; @@ -84,7 +85,7 @@ public class MutableVolumeSet implements VolumeSet { private String clusterID; private final StorageVolumeChecker volumeChecker; - private Runnable failedVolumeListener; + private VoidCallable failedVolumeListener; private StateContext context; private final StorageVolumeFactory volumeFactory; private final StorageVolume.VolumeType volumeType; @@ -132,7 +133,7 @@ public MutableVolumeSet(String dnUuid, String clusterID, initializeVolumeSet(); } - public void setFailedVolumeListener(Runnable runnable) { + public void setFailedVolumeListener(VoidCallable runnable) { failedVolumeListener = runnable; } @@ -255,7 +256,7 @@ private void handleVolumeFailures( } if (failedVolumeListener != null) { - failedVolumeListener.run(); + failedVolumeListener.call(); } // TODO: // 1. Consider stopping IO on open containers and tearing down diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index aa9c4bd953c..4cd3c8cd0a5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@ -92,6 +92,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; + +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.RECOVERING; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CLOSED_CONTAINER_IO; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_ALREADY_EXISTS; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_INTERNAL_ERROR; @@ -119,8 +121,6 @@ import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.putBlockResponseSuccess; import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest; import static org.apache.hadoop.hdds.scm.utils.ClientCommandsUtils.getReadChunkVersion; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerDataProto.State.RECOVERING; import static org.apache.hadoop.ozone.OzoneConsts.INCREMENTAL_CHUNK_LIST; import static org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; @@ -354,6 +354,15 @@ ContainerCommandResponseProto handleCreateContainer( } long containerID = request.getContainerID(); + State containerState = request.getCreateContainer().getState(); + + if (containerState != RECOVERING) { + try { + containerSet.validateContainerIsMissing(containerID, containerState); + } catch (StorageContainerException ex) { + return ContainerUtils.logAndReturnError(LOG, ex, request); + } + } ContainerLayoutVersion layoutVersion = ContainerLayoutVersion.getConfiguredVersion(conf); @@ -378,7 +387,7 @@ ContainerCommandResponseProto handleCreateContainer( try { if (containerSet.getContainer(containerID) == null) { newContainer.create(volumeSet, volumeChoosingPolicy, clusterId); - created = containerSet.addContainer(newContainer); + created = containerSet.addContainer(newContainer, RECOVERING == newContainer.getContainerState()); } else { // The create container request for an already existing container can // arrive in case the ContainerStateMachine reapplies the transaction @@ -1070,7 +1079,7 @@ private void checkContainerOpen(KeyValueContainer kvContainer) * might already be in closing state here. */ if (containerState == State.OPEN || containerState == State.CLOSING - || containerState == State.RECOVERING) { + || containerState == RECOVERING) { return; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeStore.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeStore.java index 88aeb3c174d..d9edd6d4cb0 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeStore.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeStore.java @@ -17,27 +17,22 @@ */ package org.apache.hadoop.ozone.container.metadata; -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.utils.MetadataKeyFilters; import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter; -import org.apache.hadoop.hdds.utils.db.BatchOperationHandler; import org.apache.hadoop.hdds.utils.db.DBProfile; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList; import org.apache.hadoop.ozone.container.common.interfaces.BlockIterator; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; -import org.apache.hadoop.ozone.container.common.utils.db.DatanodeDBProfile; -import org.rocksdb.InfoLogLevel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,14 +40,11 @@ import java.io.IOException; import java.util.NoSuchElementException; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE; -import static org.apache.hadoop.hdds.utils.db.DBStoreBuilder.HDDS_DEFAULT_DB_PROFILE; - /** * Implementation of the {@link DatanodeStore} interface that contains * functionality common to all more derived datanode store implementations. */ -public abstract class AbstractDatanodeStore implements DatanodeStore { +public class AbstractDatanodeStore extends AbstractRDBStore implements DatanodeStore { private Table metadataTable; @@ -68,12 +60,6 @@ public abstract class AbstractDatanodeStore implements DatanodeStore { public static final Logger LOG = LoggerFactory.getLogger(AbstractDatanodeStore.class); - private volatile DBStore store; - private final AbstractDatanodeDBDefinition dbDef; - private final ManagedColumnFamilyOptions cfOptions; - - private static DatanodeDBProfile dbProfile; - private final boolean openReadOnly; /** * Constructs the metadata store and starts the DB services. @@ -84,114 +70,64 @@ public abstract class AbstractDatanodeStore implements DatanodeStore { protected AbstractDatanodeStore(ConfigurationSource config, AbstractDatanodeDBDefinition dbDef, boolean openReadOnly) throws IOException { - - dbProfile = DatanodeDBProfile - .getProfile(config.getEnum(HDDS_DB_PROFILE, HDDS_DEFAULT_DB_PROFILE)); - - // The same config instance is used on each datanode, so we can share the - // corresponding column family options, providing a single shared cache - // for all containers on a datanode. - cfOptions = dbProfile.getColumnFamilyOptions(config); - - this.dbDef = dbDef; - this.openReadOnly = openReadOnly; - start(config); + super(dbDef, config, openReadOnly); } @Override - public void start(ConfigurationSource config) + protected DBStore initDBStore(DBStoreBuilder dbStoreBuilder, ManagedDBOptions options, ConfigurationSource config) throws IOException { - if (this.store == null) { - ManagedDBOptions options = dbProfile.getDBOptions(); - options.setCreateIfMissing(true); - options.setCreateMissingColumnFamilies(true); - - if (this.dbDef instanceof DatanodeSchemaOneDBDefinition || - this.dbDef instanceof DatanodeSchemaTwoDBDefinition) { - long maxWalSize = DBProfile.toLong(StorageUnit.MB.toBytes(2)); - options.setMaxTotalWalSize(maxWalSize); - } - - DatanodeConfiguration dc = - config.getObject(DatanodeConfiguration.class); - // Config user log files - InfoLogLevel level = InfoLogLevel.valueOf( - dc.getRocksdbLogLevel() + "_LEVEL"); - options.setInfoLogLevel(level); - options.setMaxLogFileSize(dc.getRocksdbLogMaxFileSize()); - options.setKeepLogFileNum(dc.getRocksdbLogMaxFileNum()); - - if (this.dbDef instanceof DatanodeSchemaThreeDBDefinition) { - options.setDeleteObsoleteFilesPeriodMicros( - dc.getRocksdbDeleteObsoleteFilesPeriod()); - - // For V3, all Rocksdb dir has the same "container.db" name. So use - // parentDirName(storage UUID)-dbDirName as db metrics name - this.store = DBStoreBuilder.newBuilder(config, dbDef) - .setDBOptions(options) - .setDefaultCFOptions(cfOptions) - .setOpenReadOnly(openReadOnly) - .setDBJmxBeanNameName(dbDef.getDBLocation(config).getName() + "-" + - dbDef.getName()) - .build(); - } else { - this.store = DBStoreBuilder.newBuilder(config, dbDef) - .setDBOptions(options) - .setDefaultCFOptions(cfOptions) - .setOpenReadOnly(openReadOnly) - .build(); - } + AbstractDatanodeDBDefinition dbDefinition = this.getDbDef(); + if (dbDefinition instanceof DatanodeSchemaOneDBDefinition || + dbDefinition instanceof DatanodeSchemaTwoDBDefinition) { + long maxWalSize = DBProfile.toLong(StorageUnit.MB.toBytes(2)); + options.setMaxTotalWalSize(maxWalSize); + } + DatanodeConfiguration dc = + config.getObject(DatanodeConfiguration.class); - // Use the DatanodeTable wrapper to disable the table iterator on - // existing Table implementations retrieved from the DBDefinition. - // See the DatanodeTable's Javadoc for an explanation of why this is - // necessary. - metadataTable = new DatanodeTable<>( - dbDef.getMetadataColumnFamily().getTable(this.store)); - checkTableStatus(metadataTable, metadataTable.getName()); - - // The block iterator this class returns will need to use the table - // iterator internally, so construct a block data table instance - // that does not have the iterator disabled by DatanodeTable. - blockDataTableWithIterator = - dbDef.getBlockDataColumnFamily().getTable(this.store); - - blockDataTable = new DatanodeTable<>(blockDataTableWithIterator); - checkTableStatus(blockDataTable, blockDataTable.getName()); - - if (dbDef.getFinalizeBlocksColumnFamily() != null) { - finalizeBlocksTableWithIterator = - dbDef.getFinalizeBlocksColumnFamily().getTable(this.store); - - finalizeBlocksTable = new DatanodeTable<>( - finalizeBlocksTableWithIterator); - checkTableStatus(finalizeBlocksTable, finalizeBlocksTable.getName()); - } + if (dbDefinition instanceof DatanodeSchemaThreeDBDefinition) { + options.setDeleteObsoleteFilesPeriodMicros( + dc.getRocksdbDeleteObsoleteFilesPeriod()); - if (dbDef.getLastChunkInfoColumnFamily() != null) { - lastChunkInfoTable = new DatanodeTable<>( - dbDef.getLastChunkInfoColumnFamily().getTable(this.store)); - checkTableStatus(lastChunkInfoTable, lastChunkInfoTable.getName()); - } + // For V3, all Rocksdb dir has the same "container.db" name. So use + // parentDirName(storage UUID)-dbDirName as db metrics name + dbStoreBuilder.setDBJmxBeanNameName(dbDefinition.getDBLocation(config).getName() + "-" + + dbDefinition.getName()); } - } - - @Override - public synchronized void stop() throws Exception { - if (store != null) { - store.close(); - store = null; + DBStore dbStore = dbStoreBuilder.setDBOptions(options).build(); + + // Use the DatanodeTable wrapper to disable the table iterator on + // existing Table implementations retrieved from the DBDefinition. + // See the DatanodeTable's Javadoc for an explanation of why this is + // necessary. + metadataTable = new DatanodeTable<>( + dbDefinition.getMetadataColumnFamily().getTable(dbStore)); + checkTableStatus(metadataTable, metadataTable.getName()); + + // The block iterator this class returns will need to use the table + // iterator internally, so construct a block data table instance + // that does not have the iterator disabled by DatanodeTable. + blockDataTableWithIterator = + dbDefinition.getBlockDataColumnFamily().getTable(dbStore); + + blockDataTable = new DatanodeTable<>(blockDataTableWithIterator); + checkTableStatus(blockDataTable, blockDataTable.getName()); + + if (dbDefinition.getFinalizeBlocksColumnFamily() != null) { + finalizeBlocksTableWithIterator = + dbDefinition.getFinalizeBlocksColumnFamily().getTable(dbStore); + + finalizeBlocksTable = new DatanodeTable<>( + finalizeBlocksTableWithIterator); + checkTableStatus(finalizeBlocksTable, finalizeBlocksTable.getName()); } - } - @Override - public DBStore getStore() { - return this.store; - } - - @Override - public BatchOperationHandler getBatchHandler() { - return this.store; + if (dbDefinition.getLastChunkInfoColumnFamily() != null) { + lastChunkInfoTable = new DatanodeTable<>( + dbDefinition.getLastChunkInfoColumnFamily().getTable(dbStore)); + checkTableStatus(lastChunkInfoTable, lastChunkInfoTable.getName()); + } + return dbStore; } @Override @@ -240,44 +176,6 @@ public BlockIterator getFinalizeBlockIterator(long containerID, finalizeBlocksTableWithIterator.iterator(), filter); } - @Override - public synchronized boolean isClosed() { - if (this.store == null) { - return true; - } - return this.store.isClosed(); - } - - @Override - public void close() throws IOException { - this.store.close(); - this.cfOptions.close(); - } - - @Override - public void flushDB() throws IOException { - store.flushDB(); - } - - @Override - public void flushLog(boolean sync) throws IOException { - store.flushLog(sync); - } - - @Override - public void compactDB() throws IOException { - store.compactDB(); - } - - @VisibleForTesting - public DatanodeDBProfile getDbProfile() { - return dbProfile; - } - - protected AbstractDatanodeDBDefinition getDbDef() { - return this.dbDef; - } - protected Table getBlockDataTableWithIterator() { return this.blockDataTableWithIterator; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractRDBStore.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractRDBStore.java new file mode 100644 index 00000000000..6f51a1cb4a1 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractRDBStore.java @@ -0,0 +1,135 @@ +package org.apache.hadoop.ozone.container.metadata; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.utils.db.BatchOperationHandler; +import org.apache.hadoop.hdds.utils.db.DBDefinition; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; +import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; +import org.apache.hadoop.ozone.container.common.utils.db.DatanodeDBProfile; +import org.rocksdb.InfoLogLevel; + +import java.io.IOException; + +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE; +import static org.apache.hadoop.hdds.utils.db.DBStoreBuilder.HDDS_DEFAULT_DB_PROFILE; + +/** + * Abstract Interface defining the way to interact with any rocksDB in the datanode. + * @param Generic parameter defining the schema for the DB. + */ +public abstract class AbstractRDBStore implements AbstractStore { + private final DEF dbDef; + private final ManagedColumnFamilyOptions cfOptions; + private static DatanodeDBProfile dbProfile; + private final boolean openReadOnly; + private volatile DBStore store; + + protected AbstractRDBStore(DEF dbDef, ConfigurationSource config, boolean openReadOnly) throws IOException { + dbProfile = DatanodeDBProfile.getProfile(config.getEnum(HDDS_DB_PROFILE, HDDS_DEFAULT_DB_PROFILE)); + + // The same config instance is used on each datanode, so we can share the + // corresponding column family options, providing a single shared cache + // for all containers on a datanode. + cfOptions = dbProfile.getColumnFamilyOptions(config); + this.dbDef = dbDef; + this.openReadOnly = openReadOnly; + start(config); + } + + public void start(ConfigurationSource config) + throws IOException { + if (this.store == null) { + ManagedDBOptions options = dbProfile.getDBOptions(); + options.setCreateIfMissing(true); + options.setCreateMissingColumnFamilies(true); + + DatanodeConfiguration dc = + config.getObject(DatanodeConfiguration.class); + // Config user log files + InfoLogLevel level = InfoLogLevel.valueOf( + dc.getRocksdbLogLevel() + "_LEVEL"); + options.setInfoLogLevel(level); + options.setMaxLogFileSize(dc.getRocksdbLogMaxFileSize()); + options.setKeepLogFileNum(dc.getRocksdbLogMaxFileNum()); + this.store = initDBStore(DBStoreBuilder.newBuilder(config, dbDef) + .setDBOptions(options) + .setDefaultCFOptions(cfOptions) + .setOpenReadOnly(openReadOnly), options, config); + } + } + + protected abstract DBStore initDBStore(DBStoreBuilder dbStoreBuilder, ManagedDBOptions options, + ConfigurationSource config) throws IOException; + + public synchronized void stop() throws Exception { + if (store != null) { + store.close(); + store = null; + } + } + + public DBStore getStore() { + return this.store; + } + + public synchronized boolean isClosed() { + if (this.store == null) { + return true; + } + return this.store.isClosed(); + } + + public BatchOperationHandler getBatchHandler() { + return this.store; + } + + public void close() throws IOException { + this.store.close(); + this.cfOptions.close(); + } + + public void flushDB() throws IOException { + store.flushDB(); + } + + public void flushLog(boolean sync) throws IOException { + store.flushLog(sync); + } + + public void compactDB() throws IOException { + store.compactDB(); + } + + @VisibleForTesting + public DatanodeDBProfile getDbProfile() { + return dbProfile; + } + + protected DEF getDbDef() { + return this.dbDef; + } + +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractStore.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractStore.java new file mode 100644 index 00000000000..4e8c83f79da --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractStore.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements.  See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership.  The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License.  You may obtain a copy of the License at + * + *      http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.metadata; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.utils.db.BatchOperationHandler; +import org.apache.hadoop.hdds.utils.db.DBStore; + +import java.io.Closeable; +import java.io.IOException; + +/** + * Abstract Interface for interacting with datanode databases. + */ +public interface AbstractStore extends Closeable { + + /** + * Start datanode manager. + * + * @param configuration - Configuration + * @throws IOException - Unable to start datanode store. + */ + void start(ConfigurationSource configuration) throws IOException; + + /** + * Stop datanode manager. + */ + void stop() throws Exception; + + /** + * Get datanode store. + * + * @return datanode store. + */ + @VisibleForTesting + DBStore getStore(); + + /** + * Helper to create and write batch transactions. + */ + BatchOperationHandler getBatchHandler(); + + void flushLog(boolean sync) throws IOException; + + void flushDB() throws IOException; + + void compactDB() throws IOException; + + /** + * Returns if the underlying DB is closed. This call is thread safe. + * @return true if the DB is closed. + */ + boolean isClosed(); + + default void compactionIfNeeded() throws Exception { + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStore.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStore.java index d791d9bbeab..b8283494ef5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStore.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStore.java @@ -17,22 +17,16 @@ */ package org.apache.hadoop.ozone.container.metadata; -import com.google.common.annotations.VisibleForTesting; - import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter; import org.apache.hadoop.hdds.utils.db.BatchOperation; -import org.apache.hadoop.hdds.utils.db.BatchOperationHandler; -import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList; import org.apache.hadoop.ozone.container.common.interfaces.BlockIterator; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; -import java.io.Closeable; import java.io.IOException; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_BLOCK; @@ -40,31 +34,10 @@ /** * Interface for interacting with datanode databases. */ -public interface DatanodeStore extends Closeable { +public interface DatanodeStore extends AbstractStore { String NO_SUCH_BLOCK_ERR_MSG = "Unable to find the block."; - /** - * Start datanode manager. - * - * @param configuration - Configuration - * @throws IOException - Unable to start datanode store. - */ - void start(ConfigurationSource configuration) throws IOException; - - /** - * Stop datanode manager. - */ - void stop() throws Exception; - - /** - * Get datanode store. - * - * @return datanode store. - */ - @VisibleForTesting - DBStore getStore(); - /** * A Table that keeps the block data. * @@ -100,17 +73,6 @@ public interface DatanodeStore extends Closeable { */ Table getLastChunkInfoTable(); - /** - * Helper to create and write batch transactions. - */ - BatchOperationHandler getBatchHandler(); - - void flushLog(boolean sync) throws IOException; - - void flushDB() throws IOException; - - void compactDB() throws IOException; - BlockIterator getBlockIterator(long containerID) throws IOException; @@ -120,15 +82,6 @@ BlockIterator getBlockIterator(long containerID, BlockIterator getFinalizeBlockIterator(long containerID, KeyPrefixFilter filter) throws IOException; - /** - * Returns if the underlying DB is closed. This call is thread safe. - * @return true if the DB is closed. - */ - boolean isClosed(); - - default void compactionIfNeeded() throws Exception { - } - default BlockData getBlockByID(BlockID blockID, String blockKey) throws IOException { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MasterVolumeDBDefinition.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MasterVolumeDBDefinition.java new file mode 100644 index 00000000000..633561bc812 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MasterVolumeDBDefinition.java @@ -0,0 +1,72 @@ +package org.apache.hadoop.ozone.container.metadata; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; +import org.apache.hadoop.hdds.utils.db.DBDefinition; +import org.apache.hadoop.hdds.utils.db.LongCodec; +import org.apache.hadoop.hdds.utils.db.Proto2EnumCodec; +import org.apache.hadoop.ozone.OzoneConsts; + +import java.util.Map; + +/** + * Class for defining the schema for master volume in a datanode. + */ +public final class MasterVolumeDBDefinition extends DBDefinition.WithMap { + + private static final String CONTAINER_IDS_TABLE_NAME = "containerIds"; + + public static final DBColumnFamilyDefinition + CONTAINER_IDS_TABLE = new DBColumnFamilyDefinition<>( + CONTAINER_IDS_TABLE_NAME, + LongCodec.get(), + Proto2EnumCodec.get(State.OPEN)); + + private static final Map> + COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap( + CONTAINER_IDS_TABLE); + + private static final MasterVolumeDBDefinition INSTANCE = new MasterVolumeDBDefinition(); + + public static MasterVolumeDBDefinition get() { + return INSTANCE; + } + + private MasterVolumeDBDefinition() { + super(COLUMN_FAMILIES); + } + + @Override + public String getName() { + return OzoneConsts.CONTAINER_META_DB_NAME; + } + + @Override + public String getLocationConfigKey() { + return ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR; + } + + public DBColumnFamilyDefinition getContainerIdsTable() { + return CONTAINER_IDS_TABLE; + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MasterVolumeMetadataStore.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MasterVolumeMetadataStore.java new file mode 100644 index 00000000000..f8f7a80a706 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MasterVolumeMetadataStore.java @@ -0,0 +1,88 @@ +package org.apache.hadoop.ozone.container.metadata; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; +import org.apache.hadoop.ozone.container.common.utils.BaseReferenceCountedDB; +import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedHandle; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Class for interacting with database in the master volume of a datanode. + */ +public final class MasterVolumeMetadataStore extends AbstractRDBStore + implements MetadataStore { + + private Table containerIdsTable; + private static final ConcurrentMap> INSTANCES = + new ConcurrentHashMap<>(); + + public static ReferenceCountedHandle get(ConfigurationSource conf) throws IOException { + String dbDirPath = DBStoreBuilder.getDBDirPath(MasterVolumeDBDefinition.get(), conf).getAbsolutePath(); + try { + return new ReferenceCountedHandle<>(INSTANCES.compute(dbDirPath, (k, v) -> { + if (v != null) { + v.incrementReference(); + } + if (v == null || v.isClosed()) { + try { + MasterVolumeMetadataStore masterVolumeMetadataStore = new MasterVolumeMetadataStore(conf, false); + BaseReferenceCountedDB referenceCountedDB = + new BaseReferenceCountedDB<>(masterVolumeMetadataStore, + masterVolumeMetadataStore.getStore().getDbLocation().getAbsolutePath()); + referenceCountedDB.incrementReference(); + return referenceCountedDB; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return v; + })); + } catch (UncheckedIOException e) { + throw e.getCause(); + } + } + + private MasterVolumeMetadataStore(ConfigurationSource config, boolean openReadOnly) throws IOException { + super(MasterVolumeDBDefinition.get(), config, openReadOnly); + } + + @Override + protected DBStore initDBStore(DBStoreBuilder dbStoreBuilder, ManagedDBOptions options, ConfigurationSource config) + throws IOException { + DBStore dbStore = dbStoreBuilder.build(); + this.containerIdsTable = this.getDbDef().getContainerIdsTable().getTable(dbStore); + return dbStore; + } + + @Override + public Table getContainerIdsTable() { + return containerIdsTable; + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MetadataStore.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MetadataStore.java new file mode 100644 index 00000000000..e21ee4b4321 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/MetadataStore.java @@ -0,0 +1,35 @@ +package org.apache.hadoop.ozone.container.metadata; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.utils.db.Table; + +/** + * Interface for interacting with database in the master volume of a datanode. + */ +public interface MetadataStore extends AbstractStore { + /** + * A Table that keeps the containerIds in a datanode. + * + * @return Table + */ + Table getContainerIdsTable(); +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java index 1685d1c5fe2..198ae8f65f2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java @@ -321,7 +321,7 @@ private void resolveDuplicate(KeyValueContainer existing, private void swapAndRemoveContainer(KeyValueContainer existing, KeyValueContainer toAdd) throws IOException { containerSet.removeContainer( - existing.getContainerData().getContainerID()); + existing.getContainerData().getContainerID(), false, false); containerSet.addContainer(toAdd); KeyValueContainerUtil.removeContainer(existing.getContainerData(), hddsVolume.getConf()); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java index 44884c5c290..edac2f596ea 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java @@ -80,6 +80,9 @@ public static synchronized void init( } private static boolean shouldScan(Container container) { + if (container == null) { + return false; + } long containerID = container.getContainerData().getContainerID(); if (instance == null) { LOG.debug("Skipping on demand scan for container {} since scanner was " + diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 56c42338366..dfa5c7ff59a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -24,16 +24,20 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; +import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient; import org.apache.hadoop.hdds.security.token.TokenVerifier; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.utils.HddsServerUtil; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; @@ -52,12 +56,14 @@ import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis; import org.apache.hadoop.ozone.container.common.utils.ContainerInspectorUtil; import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil; +import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedHandle; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; import org.apache.hadoop.ozone.container.common.volume.StorageVolume; import org.apache.hadoop.ozone.container.common.volume.StorageVolume.VolumeType; import org.apache.hadoop.ozone.container.common.volume.StorageVolumeChecker; import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.StaleRecoveringContainerScrubbingService; +import org.apache.hadoop.ozone.container.metadata.MasterVolumeMetadataStore; import org.apache.hadoop.ozone.container.replication.ContainerImporter; import org.apache.hadoop.ozone.container.replication.ReplicationServer; import org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig; @@ -71,6 +77,7 @@ import java.io.IOException; import java.time.Duration; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -129,6 +136,7 @@ public class OzoneContainer { private ScheduledExecutorService dbCompactionExecutorService; private final ContainerMetrics metrics; + private ReferenceCountedHandle masterVolumeMetadataStore; enum InitializingStatus { UNINITIALIZED, INITIALIZING, INITIALIZED @@ -179,12 +187,12 @@ public OzoneContainer(HddsDatanodeService hddsDatanodeService, TimeUnit.MINUTES); } } - long recoveringContainerTimeout = config.getTimeDuration( OZONE_RECOVERING_CONTAINER_TIMEOUT, OZONE_RECOVERING_CONTAINER_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS); - - containerSet = new ContainerSet(recoveringContainerTimeout); + this.masterVolumeMetadataStore = MasterVolumeMetadataStore.get(conf); + containerSet = new ContainerSet(masterVolumeMetadataStore.getStore().getContainerIdsTable(), + recoveringContainerTimeout); metadataScanner = null; metrics = ContainerMetrics.create(conf); @@ -305,7 +313,7 @@ public GrpcTlsConfig getTlsClientConfig() { * Build's container map after volume format. */ @VisibleForTesting - public void buildContainerSet() { + public void buildContainerSet() throws IOException { Iterator volumeSetIterator = volumeSet.getVolumesList() .iterator(); ArrayList volumeThreads = new ArrayList<>(); @@ -333,6 +341,14 @@ public void buildContainerSet() { for (int i = 0; i < volumeThreads.size(); i++) { volumeThreads.get(i).join(); } + try (TableIterator> itr = + containerSet.getContainerIdsTable().iterator()) { + Map containerIds = new HashMap<>(); + while (itr.hasNext()) { + containerIds.put(itr.next().getKey(), 0L); + } + containerSet.buildMissingContainerSetAndValidate(containerIds); + } } catch (InterruptedException ex) { LOG.error("Volume Threads Interrupted exception", ex); Thread.currentThread().interrupt(); @@ -529,9 +545,13 @@ public void stop() { recoveringContainerScrubbingService.shutdown(); IOUtils.closeQuietly(metrics); ContainerMetrics.remove(); + if (this.masterVolumeMetadataStore != null) { + this.masterVolumeMetadataStore.close(); + this.masterVolumeMetadataStore = null; + } } - public void handleVolumeFailures() { + public void handleVolumeFailures() throws StorageContainerException { if (containerSet != null) { containerSet.handleVolumeFailures(context); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java index f20094079c9..aaeaa21e583 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java @@ -128,7 +128,7 @@ public void importContainer(long containerID, Path tarFilePath, try (FileInputStream input = new FileInputStream(tarFilePath.toFile())) { Container container = controller.importContainer( containerData, input, packer); - containerSet.addContainer(container); + containerSet.addContainer(container, true); } } finally { importContainerProgress.remove(containerID); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java index 55df5f43b6b..0b24161aadb 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java @@ -340,6 +340,7 @@ public void testVolumeFailure() throws IOException { conSet.handleVolumeFailures(stateContext); // ContainerID1 should be removed belonging to failed volume assertNull(conSet.getContainer(containerID1)); + assertTrue(conSet.getMissingContainerSet().contains(containerID1)); // ContainerID should exist belonging to normal volume assertNotNull(conSet.getContainer(containerID)); expectedReportCount.put( diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/VoidCallable.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/VoidCallable.java new file mode 100644 index 00000000000..5f0d1704abb --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/VoidCallable.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.utils; + +/** + * Defines a functional interface to call void returning function. + */ +@FunctionalInterface +public interface VoidCallable { + void call() throws EXCEPTION_TYPE; +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index ed8d145b666..1e42241ee43 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@ -163,7 +163,8 @@ private DBStoreBuilder(ConfigurationSource configuration, OZONE_OM_DELTA_UPDATE_DATA_SIZE_MAX_LIMIT_DEFAULT, StorageUnit.BYTES); } - private void applyDBDefinition(DBDefinition definition) { + public static File getDBDirPath(DBDefinition definition, + ConfigurationSource configuration) { // Set metadata dirs. File metadataDir = definition.getDBLocation(configuration); @@ -174,6 +175,12 @@ private void applyDBDefinition(DBDefinition definition) { HddsConfigKeys.OZONE_METADATA_DIRS); metadataDir = getOzoneMetaDirPath(configuration); } + return metadataDir; + } + + private void applyDBDefinition(DBDefinition definition) { + // Set metadata dirs. + File metadataDir = getDBDirPath(definition, configuration); setName(definition.getName()); setPath(Paths.get(metadataDir.getPath())); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBTestUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBTestUtils.java new file mode 100644 index 00000000000..0eed13cbe30 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBTestUtils.java @@ -0,0 +1,142 @@ +package org.apache.hadoop.hdds.utils.db; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import org.apache.hadoop.hdds.utils.MetadataKeyFilters; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Util class for mocking DB interactions happening in various tests. + */ +public final class DBTestUtils { + + private DBTestUtils() { + + } + + public static Table getInMemoryTableForTest() { + return new Table() { + private final Map map = new ConcurrentHashMap<>(); + + @Override + public void close() { + } + + @Override + public void put(KEY key, VALUE value) { + map.put(key, value); + } + + @Override + public void putWithBatch(BatchOperation batch, KEY key, VALUE value) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public boolean isExist(KEY key) { + return map.containsKey(key); + } + + @Override + public VALUE get(KEY key) { + return map.get(key); + } + + @Override + public VALUE getIfExist(KEY key) { + return map.get(key); + } + + @Override + public void delete(KEY key) { + map.remove(key); + } + + @Override + public void deleteWithBatch(BatchOperation batch, KEY key) { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteRange(KEY beginKey, KEY endKey) { + throw new UnsupportedOperationException(); + } + + @Override + public TableIterator> iterator() { + throw new UnsupportedOperationException(); + } + + @Override + public TableIterator> iterator(KEY prefix) { + throw new UnsupportedOperationException(); + } + + @Override + public String getName() { + return ""; + } + + @Override + public long getEstimatedKeyCount() { + return map.size(); + } + + @Override + public List> getRangeKVs(KEY startKey, int count, KEY prefix, + MetadataKeyFilters.MetadataKeyFilter... filters) + throws IOException, IllegalArgumentException { + throw new UnsupportedOperationException(); + } + + @Override + public List> getSequentialRangeKVs(KEY startKey, int count, KEY prefix, + MetadataKeyFilters.MetadataKeyFilter... filters) + throws IOException, IllegalArgumentException { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteBatchWithPrefix(BatchOperation batch, KEY prefix) { + throw new UnsupportedOperationException(); + } + + @Override + public void dumpToFileWithPrefix(File externalFile, KEY prefix) { + throw new UnsupportedOperationException(); + } + + @Override + public void loadFromFile(File externalFile) { + throw new UnsupportedOperationException(); + } + }; + } +} diff --git a/hadoop-ozone/dist/src/main/compose/compatibility/docker-config b/hadoop-ozone/dist/src/main/compose/compatibility/docker-config index d3984110d8d..f7f1c24b8a0 100644 --- a/hadoop-ozone/dist/src/main/compose/compatibility/docker-config +++ b/hadoop-ozone/dist/src/main/compose/compatibility/docker-config @@ -21,7 +21,7 @@ OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-balancer/docker-config index 10d9f5c8cf5..f4866c4240d 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer/docker-config @@ -34,7 +34,7 @@ OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1 OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2 OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3 OZONE-SITE.XML_ozone.scm.ratis.enable=true -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.container.size=100MB OZONE-SITE.XML_ozone.scm.block.size=20MB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config index 623f9595583..ba4d80a9d05 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config @@ -27,7 +27,7 @@ OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config index 08c490ea51f..ebf2ce532bd 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config @@ -34,7 +34,7 @@ OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1 OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2 OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3 OZONE-SITE.XML_ozone.scm.ratis.enable=true -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config index 65834455eaa..ae2fb092be6 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config @@ -23,7 +23,7 @@ OZONE-SITE.XML_ozone.om.address.omservice.om2=om2 OZONE-SITE.XML_ozone.om.address.omservice.om3=om3 OZONE-SITE.XML_ozone.om.ratis.enable=true OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-config index 79d2e5285fb..f0ec8fcaa1a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-config @@ -24,7 +24,7 @@ OZONE-SITE.XML_ozone.om.address.omservice.om3=om3 OZONE-SITE.XML_ozone.om.ratis.enable=true OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config index 8239aad2a5d..59b1fcf8cab 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config @@ -24,7 +24,7 @@ OZONE-SITE.XML_ozone.ozone.scm.block.size=64MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index a657f22340e..f2a9e044793 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -29,7 +29,7 @@ OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon diff --git a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config index 06696a0e413..87b0cb50537 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config @@ -19,7 +19,7 @@ CORE-SITE.XML_fs.defaultFS=ofs://om OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config index 66f4cf151ec..adfaeb287d0 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config @@ -17,7 +17,7 @@ CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 OZONE-SITE.XML_ozone.ksm.address=ksm OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.block.client.address=scm diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config index 38cc5b71a18..1495e89813a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config @@ -47,7 +47,7 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config index 12a7819d1ad..2a58ffcf384 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config @@ -22,7 +22,7 @@ OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index 4f13d624969..387a1c8517e 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@ -27,7 +27,7 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed diff --git a/hadoop-ozone/dist/src/main/compose/restart/docker-config b/hadoop-ozone/dist/src/main/compose/restart/docker-config index 161af7a2975..852eb6647c3 100644 --- a/hadoop-ozone/dist/src/main/compose/restart/docker-config +++ b/hadoop-ozone/dist/src/main/compose/restart/docker-config @@ -21,7 +21,7 @@ OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config index a1b6da80c4b..d06d3279dc9 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config @@ -35,7 +35,7 @@ OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1 OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config index 88126ddf2cb..ce4a8807e54 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config @@ -25,7 +25,7 @@ OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.client.address=scm diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config index 77fa2b40ee4..a049ba5f012 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config @@ -27,7 +27,7 @@ OZONE-SITE.XML_ozone.om.ratis.enable=true OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.client.address=scm diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/docker-config b/hadoop-ozone/dist/src/main/compose/xcompat/docker-config index 1a61aaf4f7e..746b2b6e943 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/docker-config +++ b/hadoop-ozone/dist/src/main/compose/xcompat/docker-config @@ -32,7 +32,7 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index 8e72d5e5d98..99c3786233c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.container.ozoneimpl; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; @@ -31,11 +32,13 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.ozone.container.common.ContainerTestUtils; import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; +import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.HashMap; import java.util.LinkedList; @@ -158,6 +161,134 @@ public void testOzoneContainerViaDataNode() throws Exception { } } + @Test + public void testOzoneContainerWithMissingContainer() throws Exception { + MiniOzoneCluster cluster = null; + try { + long containerID = + ContainerTestHelper.getTestContainerID(); + OzoneConfiguration conf = newOzoneConfiguration(); + + // Start ozone container Via Datanode create. + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(1) + .build(); + cluster.waitForClusterToBeReady(); + + runTestOzoneContainerWithMissingContainer(cluster, containerID); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + private void runTestOzoneContainerWithMissingContainer( + MiniOzoneCluster cluster, long testContainerID) throws Exception { + ContainerProtos.ContainerCommandRequestProto + request, writeChunkRequest, putBlockRequest, + updateRequest1, updateRequest2; + ContainerProtos.ContainerCommandResponseProto response, + updateResponse1, updateResponse2; + XceiverClientGrpc client = null; + try { + // This client talks to ozone container via datanode. + client = createClientForTesting(cluster); + client.connect(); + Pipeline pipeline = client.getPipeline(); + createContainerForTesting(client, testContainerID); + writeChunkRequest = writeChunkForContainer(client, testContainerID, + 1024); + + DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0).getDatanodeDetails(); + File containerPath = + new File(cluster.getHddsDatanode(datanodeDetails).getDatanodeStateMachine() + .getContainer().getContainerSet().getContainer(testContainerID) + .getContainerData().getContainerPath()); + cluster.getHddsDatanode(datanodeDetails).stop(); + FileUtils.deleteDirectory(containerPath); + + // Restart & Check if the container has been marked as missing, since the container directory has been deleted. + cluster.restartHddsDatanode(datanodeDetails, false); + GenericTestUtils.waitFor(() -> { + try { + return cluster.getHddsDatanode(datanodeDetails).getDatanodeStateMachine() + .getContainer().getContainerSet() + .getMissingContainerSet().contains(testContainerID); + } catch (IOException e) { + return false; + } + }, 1000, 30000); + + // Read Chunk + request = ContainerTestHelper.getReadChunkRequest( + pipeline, writeChunkRequest.getWriteChunk()); + + response = client.sendCommand(request); + assertNotNull(response); + assertEquals(ContainerProtos.Result.CONTAINER_NOT_FOUND, response.getResult()); + + response = createContainerForTesting(client, testContainerID); + assertEquals(ContainerProtos.Result.CONTAINER_MISSING, response.getResult()); + + // Put Block + putBlockRequest = ContainerTestHelper.getPutBlockRequest( + pipeline, writeChunkRequest.getWriteChunk()); + + + response = client.sendCommand(putBlockRequest); + assertNotNull(response); + assertEquals(ContainerProtos.Result.CONTAINER_MISSING, response.getResult()); + + // Get Block + request = ContainerTestHelper. + getBlockRequest(pipeline, putBlockRequest.getPutBlock()); + response = client.sendCommand(request); + assertEquals(ContainerProtos.Result.CONTAINER_NOT_FOUND, response.getResult()); + + // Delete Block and Delete Chunk are handled by BlockDeletingService + // ContainerCommandRequestProto DeleteBlock and DeleteChunk requests + // are deprecated + + //Update an existing container + Map containerUpdate = new HashMap(); + containerUpdate.put("container_updated_key", "container_updated_value"); + updateRequest1 = ContainerTestHelper.getUpdateContainerRequest( + testContainerID, containerUpdate); + updateResponse1 = client.sendCommand(updateRequest1); + assertNotNull(updateResponse1); + assertEquals(ContainerProtos.Result.CONTAINER_NOT_FOUND, + response.getResult()); + + //Update an non-existing container + long nonExistingContinerID = + ContainerTestHelper.getTestContainerID(); + updateRequest2 = ContainerTestHelper.getUpdateContainerRequest( + nonExistingContinerID, containerUpdate); + updateResponse2 = client.sendCommand(updateRequest2); + assertEquals(ContainerProtos.Result.CONTAINER_NOT_FOUND, + updateResponse2.getResult()); + + // Restarting again & checking if the container is still not present on disk and marked as missing, this is to + // ensure the previous write request didn't inadvertently create the container data. + cluster.restartHddsDatanode(datanodeDetails, false); + GenericTestUtils.waitFor(() -> { + try { + return cluster.getHddsDatanode(datanodeDetails).getDatanodeStateMachine() + .getContainer().getContainerSet() + .getMissingContainerSet().contains(testContainerID); + } catch (IOException e) { + return false; + } + }, 1000, 30000); + + } finally { + if (client != null) { + client.close(); + } + } + } + public static void runTestOzoneContainerViaDataNode( long testContainerID, XceiverClientSpi client) throws Exception { ContainerProtos.ContainerCommandRequestProto @@ -504,10 +635,14 @@ private static XceiverClientGrpc createClientForTesting( MiniOzoneCluster cluster) { Pipeline pipeline = cluster.getStorageContainerManager() .getPipelineManager().getPipelines().iterator().next(); + return createClientForTesting(pipeline, cluster); + } + + private static XceiverClientGrpc createClientForTesting(Pipeline pipeline, MiniOzoneCluster cluster) { return new XceiverClientGrpc(pipeline, cluster.getConf()); } - public static void createContainerForTesting(XceiverClientSpi client, + public static ContainerProtos.ContainerCommandResponseProto createContainerForTesting(XceiverClientSpi client, long containerID) throws Exception { // Create container ContainerProtos.ContainerCommandRequestProto request = @@ -516,6 +651,7 @@ public static void createContainerForTesting(XceiverClientSpi client, ContainerProtos.ContainerCommandResponseProto response = client.sendCommand(request); assertNotNull(response); + return response; } public static ContainerProtos.ContainerCommandRequestProto diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java index 87482cb549b..7209c734104 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.utils.db.DBDefinition; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.container.metadata.MasterVolumeDBDefinition; import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaOneDBDefinition; import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaThreeDBDefinition; import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaTwoDBDefinition; @@ -56,7 +57,8 @@ private DBDefinitionFactory() { static { final Map map = new HashMap<>(); - Arrays.asList(SCMDBDefinition.get(), OMDBDefinition.get(), ReconSCMDBDefinition.get()) + Arrays.asList(SCMDBDefinition.get(), OMDBDefinition.get(), ReconSCMDBDefinition.get(), + MasterVolumeDBDefinition.get()) .forEach(dbDefinition -> map.put(dbDefinition.getName(), dbDefinition)); DB_MAP = Collections.unmodifiableMap(map); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java index 5592926bf88..47260d62f73 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java @@ -115,7 +115,7 @@ OzoneConfiguration getOzoneConf() { public void loadContainersFromVolumes() throws IOException { OzoneConfiguration conf = parent.getOzoneConf(); - ContainerSet containerSet = new ContainerSet(1000); + ContainerSet containerSet = new ContainerSet(null, 1000, true); ContainerMetrics metrics = ContainerMetrics.create(conf); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java index 393c7e599c5..2c4beb048c6 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java @@ -32,8 +32,10 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.interfaces.Handler; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; +import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedHandle; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; import org.apache.hadoop.ozone.container.common.volume.StorageVolume; +import org.apache.hadoop.ozone.container.metadata.MasterVolumeMetadataStore; import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController; import org.apache.hadoop.ozone.container.replication.ContainerImporter; import org.apache.hadoop.ozone.container.replication.ContainerReplicator; @@ -82,66 +84,74 @@ public class ClosedContainerReplicator extends BaseFreonGenerator implements private ContainerReplicator replicator; private Timer timer; + private ReferenceCountedHandle masterVolumeMetadataStoreReferenceCountedDB; private List replicationTasks; @Override public Void call() throws Exception { - OzoneConfiguration conf = createOzoneConfiguration(); + try { + OzoneConfiguration conf = createOzoneConfiguration(); - final Collection datanodeStorageDirs = - HddsServerUtil.getDatanodeStorageDirs(conf); + final Collection datanodeStorageDirs = + HddsServerUtil.getDatanodeStorageDirs(conf); - for (String dir : datanodeStorageDirs) { - checkDestinationDirectory(dir); - } + for (String dir : datanodeStorageDirs) { + checkDestinationDirectory(dir); + } - final ContainerOperationClient containerOperationClient = - new ContainerOperationClient(conf); + final ContainerOperationClient containerOperationClient = + new ContainerOperationClient(conf); - final List containerInfos = - containerOperationClient.listContainer(0L, 1_000_000).getContainerInfoList(); + final List containerInfos = + containerOperationClient.listContainer(0L, 1_000_000).getContainerInfoList(); - //logic same as the download+import on the destination datanode - initializeReplicationSupervisor(conf, containerInfos.size() * 2); + //logic same as the download+import on the destination datanode + initializeReplicationSupervisor(conf, containerInfos.size() * 2); - replicationTasks = new ArrayList<>(); + replicationTasks = new ArrayList<>(); - for (ContainerInfo container : containerInfos) { + for (ContainerInfo container : containerInfos) { - final ContainerWithPipeline containerWithPipeline = - containerOperationClient - .getContainerWithPipeline(container.getContainerID()); + final ContainerWithPipeline containerWithPipeline = + containerOperationClient + .getContainerWithPipeline(container.getContainerID()); - if (container.getState() == LifeCycleState.CLOSED) { + if (container.getState() == LifeCycleState.CLOSED) { - final List datanodesWithContainer = - containerWithPipeline.getPipeline().getNodes(); + final List datanodesWithContainer = + containerWithPipeline.getPipeline().getNodes(); - final List datanodeUUIDs = - datanodesWithContainer - .stream().map(DatanodeDetails::getUuidString) - .collect(Collectors.toList()); + final List datanodeUUIDs = + datanodesWithContainer + .stream().map(DatanodeDetails::getUuidString) + .collect(Collectors.toList()); - //if datanode is specified, replicate only container if it has a - //replica. - if (datanode.isEmpty() || datanodeUUIDs.contains(datanode)) { - replicationTasks.add(new ReplicationTask( - ReplicateContainerCommand.fromSources(container.getContainerID(), - datanodesWithContainer), replicator)); + //if datanode is specified, replicate only container if it has a + //replica. + if (datanode.isEmpty() || datanodeUUIDs.contains(datanode)) { + replicationTasks.add(new ReplicationTask( + ReplicateContainerCommand.fromSources(container.getContainerID(), + datanodesWithContainer), replicator)); + } } + } - } + //important: override the max number of tasks. + setTestNo(replicationTasks.size()); - //important: override the max number of tasks. - setTestNo(replicationTasks.size()); + init(); - init(); + timer = getMetrics().timer("replicate-container"); + runTests(this::replicateContainer); + } finally { + if (masterVolumeMetadataStoreReferenceCountedDB != null) { + masterVolumeMetadataStoreReferenceCountedDB.close(); + } - timer = getMetrics().timer("replicate-container"); - runTests(this::replicateContainer); + } return null; } @@ -173,8 +183,9 @@ private void initializeReplicationSupervisor( if (fakeDatanodeUuid.isEmpty()) { fakeDatanodeUuid = UUID.randomUUID().toString(); } - - ContainerSet containerSet = new ContainerSet(1000); + ReferenceCountedHandle referenceCountedDS = MasterVolumeMetadataStore.get(conf); + this.masterVolumeMetadataStoreReferenceCountedDB = referenceCountedDS; + ContainerSet containerSet = new ContainerSet(referenceCountedDS.getStore().getContainerIdsTable(), 1000); ContainerMetrics metrics = ContainerMetrics.create(conf);