Skip to content

Commit

Permalink
HDDS-815. Rename Ozone/HDDS config keys prefixed with 'dfs'
Browse files Browse the repository at this point in the history
  • Loading branch information
sarvekshayr committed Feb 7, 2024
1 parent 296d571 commit 2d95bb5
Show file tree
Hide file tree
Showing 49 changed files with 154 additions and 154 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ private synchronized void connectToDatanode(DatanodeDetails dn)
// port.
int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
if (port == 0) {
port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
port = config.getInt(OzoneConfigKeys.OZONE_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public static XceiverClientRatis newXceiverClientRatis(
org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline,
ConfigurationSource ozoneConf, ClientTrustManager trustManager) {
final String rpcType = ozoneConf
.get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
.get(ScmConfigKeys.OZONE_CONTAINER_RATIS_RPC_TYPE_KEY,
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
final GrpcTlsConfig tlsConfig = RatisHelper.createTlsClientConfig(new
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ public static RaftClient newRaftClient(RpcType rpcType, Pipeline pipeline,

private static RpcType getRpcType(ConfigurationSource conf) {
return SupportedRpcType.valueOfIgnoreCase(conf.get(
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
ScmConfigKeys.OZONE_CONTAINER_RATIS_RPC_TYPE_KEY,
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,12 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_DB_DIRS_PERMISSIONS =
"ozone.scm.db.dirs.permissions";

public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
= "dfs.container.ratis.enabled";
public static final String OZONE_CONTAINER_RATIS_ENABLED_KEY
= "ozone.container.ratis.enabled";
public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
= false;
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
= "dfs.container.ratis.rpc.type";
public static final String OZONE_CONTAINER_RATIS_RPC_TYPE_KEY
= "ozone.container.ratis.rpc.type";
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
= "GRPC";
public static final String
Expand All @@ -55,25 +55,25 @@ public final class ScmConfigKeys {
public static final int
DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME_DEFAULT
= 10;
public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= "dfs.container.ratis.replication.level";
public static final String OZONE_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= "ozone.container.ratis.replication.level";
public static final ReplicationLevel
DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT = ReplicationLevel.MAJORITY;
public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
= "dfs.container.ratis.num.container.op.executors";
public static final String OZONE_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
= "ozone.container.ratis.num.container.op.executors";
public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
= 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
public static final String OZONE_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"ozone.container.ratis.segment.size";
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
"64MB";
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"dfs.container.ratis.segment.preallocated.size";
public static final String OZONE_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"ozone.container.ratis.segment.preallocated.size";
public static final String
DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "4MB";
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"dfs.container.ratis.statemachinedata.sync.timeout";
OZONE_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"ozone.container.ratis.statemachinedata.sync.timeout";
public static final TimeDuration
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
TimeDuration.valueOf(10, TimeUnit.SECONDS);
Expand Down Expand Up @@ -121,14 +121,14 @@ public final class ScmConfigKeys {
DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(600000, TimeUnit.MILLISECONDS);
public static final String
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
"dfs.ratis.leader.election.minimum.timeout.duration";
OZONE_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
"ozone.ratis.leader.election.minimum.timeout.duration ";
public static final TimeDuration
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(5, TimeUnit.SECONDS);

public static final String DFS_RATIS_SNAPSHOT_THRESHOLD_KEY =
"dfs.ratis.snapshot.threshold";
public static final String OZONE_RATIS_SNAPSHOT_THRESHOLD_KEY =
"ozone.ratis.snapshot.threshold";
public static final long DFS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT = 100000;

// TODO : this is copied from OzoneConsts, may need to move to a better place
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@
@InterfaceAudience.Public
@InterfaceStability.Unstable
public final class OzoneConfigKeys {
public static final String DFS_CONTAINER_IPC_PORT =
"dfs.container.ipc";
public static final String OZONE_CONTAINER_IPC_PORT =
"ozone.container.ipc.port";
public static final int DFS_CONTAINER_IPC_PORT_DEFAULT = 9859;

public static final String OZONE_METADATA_DIRS = "ozone.metadata.dirs";
Expand All @@ -56,11 +56,11 @@ public final class OzoneConfigKeys {
* so that a mini cluster is able to launch multiple containers on a node.
*
* When set to false (default), the container port will be specified as
* {@link #DFS_CONTAINER_IPC_PORT} and the default value will be specified
* {@link #OZONE_CONTAINER_IPC_PORT} and the default value will be specified
* as {@link #DFS_CONTAINER_IPC_PORT_DEFAULT}.
*/
public static final String DFS_CONTAINER_IPC_RANDOM_PORT =
"dfs.container.ipc.random.port";
public static final String OZONE_CONTAINER_IPC_RANDOM_PORT =
"ozone.container.ipc.random.port";
public static final boolean DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT =
false;

Expand All @@ -76,8 +76,8 @@ public final class OzoneConfigKeys {
/**
* Ratis Port where containers listen to.
*/
public static final String DFS_CONTAINER_RATIS_IPC_PORT =
"dfs.container.ratis.ipc";
public static final String OZONE_CONTAINER_RATIS_IPC_PORT =
"ozone.container.ratis.ipc";
public static final int DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT = 9858;
/**
* Ratis Port where containers listen to admin requests.
Expand Down Expand Up @@ -133,9 +133,9 @@ public final class OzoneConfigKeys {
* When set to true, allocate a random free port for ozone container, so that
* a mini cluster is able to launch multiple containers on a node.
*/
public static final String DFS_CONTAINER_RATIS_IPC_RANDOM_PORT =
"dfs.container.ratis.ipc.random.port";
public static final boolean DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT =
public static final String OZONE_CONTAINER_RATIS_IPC_RANDOM_PORT =
"ozone.container.ratis.ipc.random.port";
public static final boolean OZONE_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT =
false;

public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS =
Expand Down Expand Up @@ -325,12 +325,12 @@ public final class OzoneConfigKeys {
public static final int
OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT = 10;

public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
public static final String OZONE_CONTAINER_RATIS_ENABLED_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_ENABLED_KEY;
public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY;
public static final String OZONE_CONTAINER_RATIS_RPC_TYPE_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_RPC_TYPE_KEY;
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT;
public static final String
Expand All @@ -340,35 +340,35 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME_DEFAULT
= ScmConfigKeys.
DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME_DEFAULT;
public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY;
public static final String OZONE_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_REPLICATION_LEVEL_KEY;
public static final ReplicationLevel
DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT;
public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY;
public static final String OZONE_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY;
public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
public static final String OZONE_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
public static final String OZONE_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
public static final String
DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;

// config settings to enable stateMachineData write timeout
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT;
OZONE_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
ScmConfigKeys.OZONE_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT;
public static final TimeDuration
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;

public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
public static final String OZONE_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"ozone.container.ratis.datanode.storage.dir";

public static final String DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY =
ScmConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY;
Expand Down Expand Up @@ -407,13 +407,13 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT;
public static final String
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
ScmConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY;
OZONE_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
ScmConfigKeys.OZONE_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY;
public static final TimeDuration
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT =
ScmConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT;
public static final String DFS_RATIS_SNAPSHOT_THRESHOLD_KEY =
ScmConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY;
public static final String OZONE_RATIS_SNAPSHOT_THRESHOLD_KEY =
ScmConfigKeys.OZONE_RATIS_SNAPSHOT_THRESHOLD_KEY;
public static final long DFS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT =
ScmConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT;

Expand Down
34 changes: 17 additions & 17 deletions hadoop-hdds/common/src/main/resources/ozone-default.xml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
</description>
</property>
<property>
<name>dfs.container.ipc</name>
<name>ozone.container.ipc.port</name>
<value>9859</value>
<tag>OZONE, CONTAINER, MANAGEMENT</tag>
<description>The ipc port number of container.</description>
Expand All @@ -73,7 +73,7 @@
</description>
</property>
<property>
<name>dfs.container.ipc.random.port</name>
<name>ozone.container.ipc.random.port</name>
<value>false</value>
<tag>OZONE, DEBUG, CONTAINER</tag>
<description>Allocates a random free port for ozone container. This is used
Expand All @@ -90,7 +90,7 @@
</description>
</property>
<property>
<name>dfs.container.ratis.statemachinedata.sync.timeout</name>
<name>ozone.container.ratis.statemachinedata.sync.timeout</name>
<value>10s</value>
<tag>OZONE, DEBUG, CONTAINER, RATIS</tag>
<description>Timeout for StateMachine data writes by Ratis.
Expand All @@ -102,7 +102,7 @@
<tag>OZONE, DEBUG, CONTAINER, RATIS</tag>
<description>Number of times the WriteStateMachineData op will be tried
before failing. If the value is not configured, it will default
to (hdds.ratis.rpc.slowness.timeout / dfs.container.ratis.statemachinedata.sync.timeout),
to (hdds.ratis.rpc.slowness.timeout / ozone.container.ratis.statemachinedata.sync.timeout),
which means that the WriteStatMachineData will be retried for every sync timeout until
the configured slowness timeout is hit, after which the StateMachine will close down the pipeline.

Expand Down Expand Up @@ -149,7 +149,7 @@
</description>
</property>
<property>
<name>dfs.container.ratis.datanode.storage.dir</name>
<name>ozone.container.ratis.datanode.storage.dir</name>
<value/>
<tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag>
<description>This directory is used for storing Ratis metadata like logs. If
Expand Down Expand Up @@ -223,7 +223,7 @@
</description>
</property>
<property>
<name>dfs.container.ratis.enabled</name>
<name>ozone.container.ratis.enabled</name>
<value>false</value>
<tag>OZONE, MANAGEMENT, PIPELINE, RATIS</tag>
<description>Ozone supports different kinds of replication pipelines. Ratis
Expand All @@ -232,7 +232,7 @@
</description>
</property>
<property>
<name>dfs.container.ratis.ipc</name>
<name>ozone.container.ratis.ipc</name>
<value>9858</value>
<tag>OZONE, CONTAINER, PIPELINE, RATIS</tag>
<description>The ipc port number of container for clients.</description>
Expand All @@ -250,7 +250,7 @@
<description>The ipc port number of container for server-server communication.</description>
</property>
<property>
<name>dfs.container.ratis.ipc.random.port</name>
<name>ozone.container.ratis.ipc.random.port</name>
<value>false</value>
<tag>OZONE,DEBUG</tag>
<description>Allocates a random free port for ozone ratis port for the
Expand All @@ -259,7 +259,7 @@
</description>
</property>
<property>
<name>dfs.container.ratis.rpc.type</name>
<name>ozone.container.ratis.rpc.type</name>
<value>GRPC</value>
<tag>OZONE, RATIS, MANAGEMENT</tag>
<description>Ratis supports different kinds of transports like netty, GRPC,
Expand All @@ -268,7 +268,7 @@
</description>
</property>
<property>
<name>dfs.ratis.snapshot.threshold</name>
<name>ozone.ratis.snapshot.threshold</name>
<value>10000</value>
<tag>OZONE, RATIS</tag>
<description>Number of transactions after which a ratis snapshot should be
Expand All @@ -281,7 +281,7 @@
<tag>OZONE, RATIS</tag>
<description>Maximum number of pending apply transactions in a data
pipeline. The default value is kept same as default snapshot threshold
dfs.ratis.snapshot.threshold.
ozone.ratis.snapshot.threshold.
</description>
</property>
<property>
Expand All @@ -303,7 +303,7 @@
</description>
</property>
<property>
<name>dfs.container.ratis.replication.level</name>
<name>ozone.container.ratis.replication.level</name>
<value>MAJORITY</value>
<tag>OZONE, RATIS</tag>
<description>Replication level to be used by datanode for submitting a
Expand All @@ -312,23 +312,23 @@
</description>
</property>
<property>
<name>dfs.container.ratis.num.container.op.executors</name>
<name>ozone.container.ratis.num.container.op.executors</name>
<value>10</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
<description>Number of executors that will be used by Ratis to execute
container ops.(10 by default).
</description>
</property>
<property>
<name>dfs.container.ratis.segment.size</name>
<name>ozone.container.ratis.segment.size</name>
<value>64MB</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
<description>The size of the raft segment file used
by Apache Ratis on datanodes. (64 MB by default)
</description>
</property>
<property>
<name>dfs.container.ratis.segment.preallocated.size</name>
<name>ozone.container.ratis.segment.preallocated.size</name>
<value>4MB</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
<description>The pre-allocated file size for raft segment used
Expand All @@ -342,7 +342,7 @@
<description>Retry Cache entry timeout for ratis server.</description>
</property>
<property>
<name>dfs.ratis.leader.election.minimum.timeout.duration</name>
<name>ozone.ratis.leader.election.minimum.timeout.duration </name>
<value>5s</value>
<tag>OZONE, RATIS, MANAGEMENT</tag>
<description>The minimum timeout duration for ratis leader election.
Expand Down Expand Up @@ -707,7 +707,7 @@

For production clusters or any time you care about performance, it is
recommended that ozone.om.db.dirs, ozone.scm.db.dirs and
dfs.container.ratis.datanode.storage.dir be configured separately.
ozone.container.ratis.datanode.storage.dir be configured separately.
</description>
</property>
<property>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,10 +99,10 @@ public XceiverServerGrpc(DatanodeDetails datanodeDetails,

this.id = datanodeDetails.getUuid();
this.datanodeDetails = datanodeDetails;
this.port = conf.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
this.port = conf.getInt(OzoneConfigKeys.OZONE_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);

if (conf.getBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT,
if (conf.getBoolean(OzoneConfigKeys.OZONE_CONTAINER_IPC_RANDOM_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT)) {
this.port = 0;
}
Expand Down
Loading

0 comments on commit 2d95bb5

Please sign in to comment.